diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 570c07cb09..a0b89ef95d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -959,7 +959,7 @@ object QueryPlanSerde extends Logging with CometExprShim { CometConf.COMET_RESPECT_PARQUET_FILTER_PUSHDOWN.get(conf)) { val dataFilters = new ListBuffer[Expr]() - for (filter <- scan.dataFilters) { + for (filter <- scan.supportedDataFilters) { exprToProto(filter, scan.output) match { case Some(proto) => dataFilters += proto case _ => diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala index 4dd889d231..efc54ab712 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala @@ -155,16 +155,21 @@ case class CometScanExec( } } - @transient - private lazy val pushedDownFilters = { - val supportedFilters = if (scanImpl == CometConf.SCAN_NATIVE_DATAFUSION) { - // `native_datafusion` scan does not support subquery pushdown filters, - // see: https://github.com/apache/datafusion-comet/issues/2424 + /** + * Returns the data filters that are supported for this scan implementation. For + * native_datafusion scans, this excludes dynamic pruning filters (subqueries) + */ + lazy val supportedDataFilters: Seq[Expression] = { + if (scanImpl == CometConf.SCAN_NATIVE_DATAFUSION) { dataFilters.filterNot(isDynamicPruningFilter) } else { dataFilters } - getPushedDownFilters(relation, supportedFilters) + } + + @transient + private lazy val pushedDownFilters = { + getPushedDownFilters(relation, supportedDataFilters) } override lazy val metadata: Map[String, String] = diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt index a1697ca9eb..1bb91d6ff3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* ColumnarToRow (40) -+- CometTakeOrderedAndProject (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) +* CometColumnarToRow (41) ++- CometTakeOrderedAndProject (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) :- CometProject (33) : +- CometBroadcastHashJoin (32) : :- CometProject (27) @@ -14,11 +14,11 @@ : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (25) : : +- CometFilter (24) : : +- CometHashAggregate (23) @@ -30,28 +30,36 @@ : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (13) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (13) : : +- ReusedExchange (15) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- CometBroadcastExchange (36) - +- CometFilter (35) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (34) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.customer (34) -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(1) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#4)] +PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_year#6] @@ -92,9 +100,13 @@ Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] Condition : isnotnull(ctr_total_return#10) -(13) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(13) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] -Arguments: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct (14) CometFilter Input [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] @@ -157,13 +169,16 @@ Arguments: [ctr_store_sk#9], [ctr_store_sk#17], Inner, (cast(ctr_total_return#10 Input [5]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10, (avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] Arguments: [ctr_customer_sk#8, ctr_store_sk#9], [ctr_customer_sk#8, ctr_store_sk#9] -(28) CometNativeScan: `spark_catalog`.`default`.`store` +(28) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#22, s_state#23] -Arguments: [s_store_sk#22, s_state#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (29) CometFilter Input [2]: [s_store_sk#22, s_state#23] -Condition : ((isnotnull(s_state#23) AND (s_state#23 = TN)) AND isnotnull(s_store_sk#22)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#23, 2, true, false, true) = TN) AND isnotnull(s_store_sk#22)) (30) CometProject Input [2]: [s_store_sk#22, s_state#23] @@ -182,31 +197,38 @@ Arguments: [ctr_store_sk#9], [s_store_sk#22], Inner, BuildRight Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, s_store_sk#22] Arguments: [ctr_customer_sk#8], [ctr_customer_sk#8] -(34) CometNativeScan: `spark_catalog`.`default`.`customer` +(34) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_sk#24, c_customer_id#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (35) CometFilter Input [2]: [c_customer_sk#24, c_customer_id#25] Condition : isnotnull(c_customer_sk#24) -(36) CometBroadcastExchange +(36) CometProject Input [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_sk#24, c_customer_id#25] +Arguments: [c_customer_sk#24, c_customer_id#26], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#26] -(37) CometBroadcastHashJoin +(37) CometBroadcastExchange +Input [2]: [c_customer_sk#24, c_customer_id#26] +Arguments: [c_customer_sk#24, c_customer_id#26] + +(38) CometBroadcastHashJoin Left output [1]: [ctr_customer_sk#8] -Right output [2]: [c_customer_sk#24, c_customer_id#25] +Right output [2]: [c_customer_sk#24, c_customer_id#26] Arguments: [ctr_customer_sk#8], [c_customer_sk#24], Inner, BuildRight -(38) CometProject -Input [3]: [ctr_customer_sk#8, c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_id#25], [c_customer_id#25] +(39) CometProject +Input [3]: [ctr_customer_sk#8, c_customer_sk#24, c_customer_id#26] +Arguments: [c_customer_id#26], [c_customer_id#26] -(39) CometTakeOrderedAndProject -Input [1]: [c_customer_id#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#25 ASC NULLS FIRST], output=[c_customer_id#25]), [c_customer_id#25], 100, [c_customer_id#25 ASC NULLS FIRST], [c_customer_id#25] +(40) CometTakeOrderedAndProject +Input [1]: [c_customer_id#26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#26 ASC NULLS FIRST], output=[c_customer_id#26]), [c_customer_id#26], 100, 0, [c_customer_id#26 ASC NULLS FIRST], [c_customer_id#26] -(40) ColumnarToRow [codegen id : 1] -Input [1]: [c_customer_id#25] +(41) CometColumnarToRow [codegen id : 1] +Input [1]: [c_customer_id#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/extended.txt new file mode 100644 index 0000000000..9470bbb18b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/extended.txt @@ -0,0 +1,46 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 43 out of 43 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/simplified.txt index 87657df444..ac53cb7164 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_customer_id] CometProject [c_customer_id] @@ -9,34 +9,35 @@ WholeStageCodegen (1) CometProject [ctr_customer_sk,ctr_store_sk] CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_store_sk] CometFilter [ctr_customer_sk,ctr_store_sk,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] + CometHashAggregate [sum] [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum(UnscaledValue(sr_return_amt))] CometExchange [sr_customer_sk,sr_store_sk] #1 - CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #3 CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count,avg(ctr_total_return)] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_store_sk,avg(ctr_total_return)] CometExchange [ctr_store_sk] #4 - CometHashAggregate [ctr_store_sk,sum,count,ctr_total_return] - CometHashAggregate [ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] + CometHashAggregate [ctr_total_return] [ctr_store_sk,sum,count] + CometHashAggregate [sr_customer_sk,sum] [ctr_store_sk,ctr_total_return,sr_store_sk,sum(UnscaledValue(sr_return_amt))] CometExchange [sr_customer_sk,sr_store_sk] #5 - CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] ReusedExchange [d_date_sk] #2 CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [c_customer_sk,c_customer_id] #7 - CometFilter [c_customer_sk,c_customer_id] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id] + CometProject [c_customer_id] [c_customer_sk,c_customer_id] + CometFilter [c_customer_sk,c_customer_id] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/explain.txt index 7c92771145..29c99eb746 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/explain.txt @@ -1,61 +1,77 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (28) - : +- * ColumnarToRow (27) - : +- CometProject (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (34) - +- * ColumnarToRow (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] @@ -87,12 +103,15 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#13] @@ -106,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -119,96 +138,132 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct -(21) BroadcastHashJoin [codegen id : 5] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None -(22) Filter [codegen id : 5] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(23) Project [codegen id : 5] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15, ca_county#16] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_county#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] +ReadSchema: struct -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_county#16] -Condition : (ca_county#16 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#15)) +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) -(26) CometProject -Input [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15], [ca_address_sk#15] +(31) CometProject +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(27) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#15] +(32) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#17] -(28) BroadcastExchange -Input [1]: [ca_address_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(33) BroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(29) BroadcastHashJoin [codegen id : 5] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(30) Project [codegen id : 5] +(35) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] -(31) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(32) CometFilter -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Condition : isnotnull(cd_demo_sk#17) +(37) CometFilter +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) -(33) ColumnarToRow [codegen id : 4] -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(38) CometProject +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#20, 1, true, false, true) AS cd_gender#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) AS cd_education_status#30, cd_purchase_estimate#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#24, 10, true, false, true) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(34) BroadcastExchange -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(39) CometColumnarToRow [codegen id : 4] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(35) BroadcastHashJoin [codegen id : 5] +(40) BroadcastExchange +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] +Right keys [1]: [cd_demo_sk#19] Join type: Inner Join condition: None -(36) Project [codegen id : 5] -Output [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(42) Project [codegen id : 5] +Output [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(37) HashAggregate [codegen id : 5] -Input [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(43) HashAggregate [codegen id : 5] +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#26] -Results [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] +Aggregate Attributes [1]: [count#32] +Results [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] + +(44) CometColumnarExchange +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(38) Exchange -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Arguments: hashpartitioning(cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(45) CometColumnarToRow [codegen id : 6] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] -(39) HashAggregate [codegen id : 6] -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(46) HashAggregate [codegen id : 6] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#28] -Results [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, count(1)#28 AS cnt1#29, cd_purchase_estimate#21, count(1)#28 AS cnt2#30, cd_credit_rating#22, count(1)#28 AS cnt3#31, cd_dep_count#23, count(1)#28 AS cnt4#32, cd_dep_employed_count#24, count(1)#28 AS cnt5#33, cd_dep_college_count#25, count(1)#28 AS cnt6#34] +Aggregate Attributes [1]: [count(1)#34] +Results [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, count(1)#34 AS cnt1#35, cd_purchase_estimate#23, count(1)#34 AS cnt2#36, cd_credit_rating#31, count(1)#34 AS cnt3#37, cd_dep_count#25, count(1)#34 AS cnt4#38, cd_dep_employed_count#26, count(1)#34 AS cnt5#39, cd_dep_college_count#27, count(1)#34 AS cnt6#40] -(40) TakeOrderedAndProject -Input [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] -Arguments: 100, [cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_education_status#20 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#22 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] +(47) TakeOrderedAndProject +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] +Arguments: 100, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/simplified.txt index 228afd2bac..e5aeaa20c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/simplified.txt @@ -1,54 +1,63 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/explain.txt index d4ba47247d..d9afb2f640 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/explain.txt @@ -1,331 +1,388 @@ == Physical Plan == -* ColumnarToRow (63) -+- CometTakeOrderedAndProject (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (47) - : +- CometBroadcastHashJoin (46) - : :- CometProject (32) - : : +- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (45) - : +- CometFilter (44) - : +- CometHashAggregate (43) - : +- CometExchange (42) - : +- CometHashAggregate (41) - : +- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometFilter (34) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (33) - : : +- ReusedExchange (35) - : +- ReusedExchange (38) - +- CometBroadcastExchange (59) - +- CometHashAggregate (58) - +- CometExchange (57) - +- CometHashAggregate (56) - +- CometProject (55) - +- CometBroadcastHashJoin (54) - :- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometFilter (49) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (48) - : +- ReusedExchange (50) - +- ReusedExchange (53) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (69) ++- CometTakeOrderedAndProject (68) + +- CometProject (67) + +- CometBroadcastHashJoin (66) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (34) + : : +- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (50) + : +- CometFilter (49) + : +- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometProject (37) + : : : +- CometFilter (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (35) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- CometBroadcastExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometProject (55) + : : +- CometFilter (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (53) + : +- ReusedExchange (56) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(13) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(14) CometExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(16) CometFilter -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Arguments: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] - -(18) CometFilter -Input [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_customer_id#19)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Right output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_sk#18], [ss_customer_sk#26], Inner, BuildRight - -(21) CometProject -Input [12]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(23) CometFilter -Input [2]: [d_date_sk#30, d_year#31] -Condition : ((isnotnull(d_year#31) AND (d_year#31 = 2002)) AND isnotnull(d_date_sk#30)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(25) CometBroadcastHashJoin -Left output [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Right output [2]: [d_date_sk#30, d_year#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight - -(26) CometProject -Input [12]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#30, d_year#31] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] - -(27) CometHashAggregate -Input [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(28) CometExchange -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Arguments: hashpartitioning(c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(29) CometHashAggregate -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(30) CometBroadcastExchange -Input [3]: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#16, year_total#17] -Right output [3]: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#16], [customer_id#33], Inner, BuildRight - -(32) CometProject -Input [5]: [customer_id#16, year_total#17, customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35], [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35] - -(33) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Arguments: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] - -(34) CometFilter -Input [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Condition : (isnotnull(c_customer_sk#36) AND isnotnull(c_customer_id#37)) - -(35) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) + +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] + +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] + +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight + +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] + +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight + +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] -(36) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Right output [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Arguments: [c_customer_sk#36], [ws_bill_customer_sk#44], Inner, BuildRight +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometProject +Input [5]: [customer_id#22, year_total#23, customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] + +(35) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(36) CometFilter +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Condition : (isnotnull(c_customer_sk#42) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true))) (37) CometProject -Input [12]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Arguments: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47], [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] - -(38) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#48, d_year#49] - -(39) CometBroadcastHashJoin -Left output [10]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Right output [2]: [d_date_sk#48, d_year#49] -Arguments: [ws_sold_date_sk#47], [d_date_sk#48], Inner, BuildRight - -(40) CometProject -Input [12]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47, d_date_sk#48, d_year#49] -Arguments: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49], [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49] - -(41) CometHashAggregate -Input [10]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49] -Keys [8]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#46 - ws_ext_discount_amt#45)))] - -(42) CometExchange -Input [9]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, sum#50] -Arguments: hashpartitioning(c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(43) CometHashAggregate -Input [9]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, sum#50] -Keys [8]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#46 - ws_ext_discount_amt#45)))] - -(44) CometFilter -Input [2]: [customer_id#51, year_total#52] -Condition : (isnotnull(year_total#52) AND (year_total#52 > 0.00)) - -(45) CometBroadcastExchange -Input [2]: [customer_id#51, year_total#52] -Arguments: [customer_id#51, year_total#52] - -(46) CometBroadcastHashJoin -Left output [4]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35] -Right output [2]: [customer_id#51, year_total#52] -Arguments: [customer_id#16], [customer_id#51], Inner, BuildRight - -(47) CometProject -Input [6]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, customer_id#51, year_total#52] -Arguments: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52], [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52] - -(48) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Arguments: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Arguments: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55], [c_customer_sk#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#44, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#45, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#46, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#48, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#49, 50, true, false, true) AS c_email_address#55] + +(38) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#59)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(39) CometFilter +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Condition : isnotnull(ws_bill_customer_sk#56) + +(40) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] + +(41) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#42], [ws_bill_customer_sk#56], Inner, BuildRight + +(42) CometProject +Input [12]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] + +(43) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#60, d_year#61] + +(44) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(45) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] + +(46) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] + +(47) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(48) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] (49) CometFilter -Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Condition : (isnotnull(c_customer_sk#53) AND isnotnull(c_customer_id#54)) +Input [2]: [customer_id#63, year_total#64] +Condition : (isnotnull(year_total#64) AND (year_total#64 > 0.00)) -(50) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] +(50) CometBroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#63, year_total#64] (51) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Right output [4]: [ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Arguments: [c_customer_sk#53], [ws_bill_customer_sk#61], Inner, BuildRight +Left output [4]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] +Right output [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#22], [customer_id#63], Inner, BuildRight (52) CometProject -Input [12]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] +Input [6]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, customer_id#63, year_total#64] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] -(53) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#65, d_year#66] +(53) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(54) CometBroadcastHashJoin -Left output [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [ws_sold_date_sk#64], [d_date_sk#65], Inner, BuildRight +(54) CometFilter +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Condition : (isnotnull(c_customer_sk#65) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true))) (55) CometProject -Input [12]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64, d_date_sk#65, d_year#66] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66] +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Arguments: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55], [c_customer_sk#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#67, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#68, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#69, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#71, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#72, 50, true, false, true) AS c_email_address#55] -(56) CometHashAggregate -Input [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#63 - ws_ext_discount_amt#62)))] +(56) ReusedExchange [Reuses operator id: 40] +Output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(57) CometExchange -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, sum#67] -Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(57) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_sk#65], [ws_bill_customer_sk#73], Inner, BuildRight -(58) CometHashAggregate -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, sum#67] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#63 - ws_ext_discount_amt#62)))] +(58) CometProject +Input [12]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(59) CometBroadcastExchange -Input [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#68, year_total#69] +(59) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#77, d_year#78] (60) CometBroadcastHashJoin -Left output [5]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52] -Right output [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#16], [customer_id#68], Inner, (CASE WHEN (year_total#52 > 0.00) THEN (year_total#69 / year_total#52) END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#35 / year_total#17) END), BuildRight +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Right output [2]: [d_date_sk#77, d_year#78] +Arguments: [ws_sold_date_sk#76], [d_date_sk#77], Inner, BuildRight (61) CometProject -Input [7]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52, customer_id#68, year_total#69] -Arguments: [customer_preferred_cust_flag#34], [customer_preferred_cust_flag#34] - -(62) CometTakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#34 ASC NULLS FIRST], output=[customer_preferred_cust_flag#34]), [customer_preferred_cust_flag#34], 100, [customer_preferred_cust_flag#34 ASC NULLS FIRST], [customer_preferred_cust_flag#34] - -(63) ColumnarToRow [codegen id : 1] -Input [1]: [customer_preferred_cust_flag#34] +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76, d_date_sk#77, d_year#78] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] + +(62) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] + +(63) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(64) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] + +(65) CometBroadcastExchange +Input [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#80, year_total#81] + +(66) CometBroadcastHashJoin +Left output [5]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] +Right output [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#22], [customer_id#80], Inner, (CASE WHEN (year_total#64 > 0.00) THEN (year_total#81 / year_total#64) END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#41 / year_total#23) END), BuildRight + +(67) CometProject +Input [7]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64, customer_id#80, year_total#81] +Arguments: [customer_preferred_cust_flag#40], [customer_preferred_cust_flag#40] + +(68) CometTakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#40] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#40 ASC NULLS FIRST], output=[customer_preferred_cust_flag#40]), [customer_preferred_cust_flag#40], 100, 0, [customer_preferred_cust_flag#40 ASC NULLS FIRST], [customer_preferred_cust_flag#40] + +(69) CometColumnarToRow [codegen id : 1] +Input [1]: [customer_preferred_cust_flag#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/extended.txt new file mode 100644 index 0000000000..1c5f33227c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/extended.txt @@ -0,0 +1,79 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 76 out of 76 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/simplified.txt index 7bfc93c3e9..b69b7cc000 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_preferred_cust_flag] CometProject [customer_preferred_cust_flag] @@ -9,57 +9,63 @@ WholeStageCodegen (1) CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_preferred_cust_flag,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #4 - CometHashAggregate [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 - CometBroadcastExchange [customer_id,year_total] #9 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #10 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/explain.txt index c6a8dff14a..7fff9bfcea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/explain.txt @@ -1,116 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/extended.txt new file mode 100644 index 0000000000..9b405bc9ee --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/simplified.txt index 64d31a68c7..3f383800c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] CometExchange [i_class] #1 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/explain.txt index 3e48fad3a2..260585cee3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/explain.txt @@ -1,12 +1,12 @@ == Physical Plan == -* ColumnarToRow (33) -+- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (34) ++- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (19) : : +- CometBroadcastHashJoin (18) : : :- CometProject (13) @@ -14,37 +14,45 @@ : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) + : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) + +- CometBroadcastExchange (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.household_demographics (26) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] +ReadSchema: struct (2) CometFilter Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometNativeScan: `spark_catalog`.`default`.`store` +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [s_store_sk#11] @@ -63,112 +71,128 @@ Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (VA,TX,MS))) (10) CometProject Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) AS ca_state#15] (11) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] (12) CometBroadcastHashJoin Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight (13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#16, d_year#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#10], [d_date_sk#15], Inner, BuildRight +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(20) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) - -(22) CometBroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#17], Inner, ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(24) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] +(22) CometProject +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(25) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(26) CometFilter -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) - -(27) CometBroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] - -(28) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Right output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#20], Inner, (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))), BuildRight - -(29) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] +(24) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight + +(25) CometProject +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] + +(26) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#23, hd_dep_count#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] +ReadSchema: struct + +(27) CometFilter +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) + +(28) CometBroadcastExchange +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] + +(29) CometBroadcastHashJoin +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight + +(30) CometProject +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(30) CometHashAggregate +(31) CometHashAggregate Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -(31) CometExchange -Input [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] +(32) CometExchange +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) CometHashAggregate -Input [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] +(33) CometHashAggregate +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -(33) ColumnarToRow [codegen id : 1] -Input [4]: [avg(ss_quantity)#29, avg(ss_ext_sales_price)#30, avg(ss_ext_wholesale_cost)#31, sum(ss_ext_wholesale_cost)#32] +(34) CometColumnarToRow [codegen id : 1] +Input [4]: [avg(ss_quantity)#32, avg(ss_ext_sales_price)#33, avg(ss_ext_wholesale_cost)#34, sum(ss_ext_wholesale_cost)#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/extended.txt new file mode 100644 index 0000000000..b69894e484 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/extended.txt @@ -0,0 +1,36 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_demographics + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.household_demographics + +Comet accelerated 33 out of 33 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/simplified.txt index ec41eb52c3..bd03c2152d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum,avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] + CometHashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] CometExchange #1 - CometHashAggregate [sum,count,sum,count,sum,count,sum,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] + CometHashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum] CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] @@ -15,21 +15,22 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometProject [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/explain.txt index c271ba7ee3..ff9a77f0ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/explain.txt @@ -1,30 +1,30 @@ == Physical Plan == -* ColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometHashAggregate (70) - +- CometExchange (69) - +- CometHashAggregate (68) - +- CometExpand (67) - +- CometUnion (66) - :- CometProject (57) - : +- CometFilter (56) - : +- CometHashAggregate (55) - : +- CometExchange (54) - : +- CometHashAggregate (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) +* CometColumnarToRow (102) ++- CometTakeOrderedAndProject (101) + +- CometHashAggregate (100) + +- CometExchange (99) + +- CometHashAggregate (98) + +- CometExpand (97) + +- CometUnion (96) + :- CometProject (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) @@ -33,89 +33,140 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - :- CometProject (61) - : +- CometFilter (60) - : +- CometHashAggregate (59) - : +- ReusedExchange (58) - +- CometProject (65) - +- CometFilter (64) - +- CometHashAggregate (63) - +- ReusedExchange (62) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + :- CometProject (80) + : +- CometFilter (79) + : +- CometHashAggregate (78) + : +- CometExchange (77) + : +- CometHashAggregate (76) + : +- CometProject (75) + : +- CometBroadcastHashJoin (74) + : :- CometProject (72) + : : +- CometBroadcastHashJoin (71) + : : :- CometBroadcastHashJoin (69) + : : : :- CometFilter (67) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (66) + : : : +- ReusedExchange (68) + : : +- ReusedExchange (70) + : +- ReusedExchange (73) + +- CometProject (95) + +- CometFilter (94) + +- CometHashAggregate (93) + +- CometExchange (92) + +- CometHashAggregate (91) + +- CometProject (90) + +- CometBroadcastHashJoin (89) + :- CometProject (87) + : +- CometBroadcastHashJoin (86) + : :- CometBroadcastHashJoin (84) + : : :- CometFilter (82) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (85) + +- ReusedExchange (88) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometNativeScan: `spark_catalog`.`default`.`item` +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometNativeScan: `spark_catalog`.`default`.`item` +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] @@ -134,9 +185,12 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [d_date_sk#21, d_year#22] @@ -207,268 +261,420 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct -(34) CometBroadcastHashJoin -Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) -(35) CometBroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [brand_id#24, class_id#25, category_id#26] +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Right output [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : isnotnull(i_item_sk#31) +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight +(43) CometBroadcastExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +(45) CometProject +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35, d_year#36, d_moy#37] +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(48) CometFilter -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 11)) AND isnotnull(d_date_sk#35)) +(48) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(49) CometProject -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35], [d_date_sk#35] +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct (56) CometFilter -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#41, number_sales#42] -Condition : (isnotnull(sales#41) AND (cast(sales#41 as decimal(32,6)) > cast(Subquery scalar-subquery#43, [id=#44] as decimal(32,6)))) +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#41, number_sales#42] -Arguments: [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48], [sales#41, number_sales#42, store AS channel#45, i_brand_id#32 AS i_brand_id#46, i_class_id#33 AS i_class_id#47, i_category_id#34 AS i_category_id#48] +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] -(58) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#49, i_class_id#50, i_category_id#51, sum#52, isEmpty#53, count#54] +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] -(59) CometHashAggregate -Input [6]: [i_brand_id#49, i_class_id#50, i_category_id#51, sum#52, isEmpty#53, count#54] -Keys [3]: [i_brand_id#49, i_class_id#50, i_category_id#51] -Functions [2]: [sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), count(1)] +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(60) CometFilter -Input [5]: [i_brand_id#49, i_class_id#50, i_category_id#51, sales#57, number_sales#58] -Condition : (isnotnull(sales#57) AND (cast(sales#57 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#43, [id=#44] as decimal(32,6)))) +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(61) CometProject -Input [5]: [i_brand_id#49, i_class_id#50, i_category_id#51, sales#57, number_sales#58] -Arguments: [sales#57, number_sales#58, channel#59, i_brand_id#49, i_class_id#50, i_category_id#51], [sales#57, number_sales#58, catalog AS channel#59, i_brand_id#49, i_class_id#50, i_category_id#51] +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#63, isEmpty#64, count#65] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#63, isEmpty#64, count#65] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), count(1)] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [5]: [i_brand_id#60, i_class_id#61, i_category_id#62, sales#68, number_sales#69] -Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#43, [id=#44] as decimal(32,6)))) +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (65) CometProject -Input [5]: [i_brand_id#60, i_class_id#61, i_category_id#62, sales#68, number_sales#69] -Arguments: [sales#68, number_sales#69, channel#70, i_brand_id#60, i_class_id#61, i_category_id#62], [sales#68, number_sales#69, web AS channel#70, i_brand_id#60, i_class_id#61, i_category_id#62] - -(66) CometUnion -Child 0 Input [6]: [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48] -Child 1 Input [6]: [sales#57, number_sales#58, channel#59, i_brand_id#49, i_class_id#50, i_category_id#51] -Child 2 Input [6]: [sales#68, number_sales#69, channel#70, i_brand_id#60, i_class_id#61, i_category_id#62] - -(67) CometExpand -Input [6]: [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48] -Arguments: [[sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48, 0], [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, null, 1], [sales#41, number_sales#42, channel#45, i_brand_id#46, null, null, 3], [sales#41, number_sales#42, channel#45, null, null, null, 7], [sales#41, number_sales#42, null, null, null, null, 15]], [sales#41, number_sales#42, channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] - -(68) CometHashAggregate -Input [7]: [sales#41, number_sales#42, channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] -Keys [5]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] -Functions [2]: [partial_sum(sales#41), partial_sum(number_sales#42)] - -(69) CometExchange -Input [8]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75, sum#76, isEmpty#77, sum#78] -Arguments: hashpartitioning(channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(70) CometHashAggregate -Input [8]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75, sum#76, isEmpty#77, sum#78] -Keys [5]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] -Functions [2]: [sum(sales#41), sum(number_sales#42)] - -(71) CometTakeOrderedAndProject -Input [6]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#71 ASC NULLS FIRST,i_brand_id#72 ASC NULLS FIRST,i_class_id#73 ASC NULLS FIRST,i_category_id#74 ASC NULLS FIRST], output=[channel#71,i_brand_id#72,i_class_id#73,i_category_id#74,sum(sales)#79,sum(number_sales)#80]), [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80], 100, [channel#71 ASC NULLS FIRST, i_brand_id#72 ASC NULLS FIRST, i_class_id#73 ASC NULLS FIRST, i_category_id#74 ASC NULLS FIRST], [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80] - -(72) ColumnarToRow [codegen id : 1] -Input [6]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#43, [id=#44] -* ColumnarToRow (89) -+- CometHashAggregate (88) - +- CometExchange (87) - +- CometHashAggregate (86) - +- CometUnion (85) - :- CometProject (76) - : +- CometBroadcastHashJoin (75) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (73) - : +- ReusedExchange (74) - :- CometProject (80) - : +- CometBroadcastHashJoin (79) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (77) - : +- ReusedExchange (78) - +- CometProject (84) - +- CometBroadcastHashJoin (83) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (81) - +- ReusedExchange (82) - - -(73) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83] -Arguments: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83] - -(74) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#84] - -(75) CometBroadcastHashJoin -Left output [3]: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83] -Right output [1]: [d_date_sk#84] -Arguments: [ss_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight - -(76) CometProject -Input [4]: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83, d_date_sk#84] -Arguments: [quantity#85, list_price#86], [ss_quantity#81 AS quantity#85, ss_list_price#82 AS list_price#86] - -(77) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89] -Arguments: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89] - -(78) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#90] - -(79) CometBroadcastHashJoin -Left output [3]: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89] -Right output [1]: [d_date_sk#90] -Arguments: [cs_sold_date_sk#89], [d_date_sk#90], Inner, BuildRight +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] + +(66) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(67) CometFilter +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) + +(68) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#57] + +(69) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#57] +Arguments: [cs_item_sk#53], [ss_item_sk#57], LeftSemi, BuildRight + +(70) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] + +(71) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_item_sk#53], [i_item_sk#58], Inner, BuildRight + +(72) CometProject +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] + +(73) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#62] + +(74) CometBroadcastHashJoin +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] +Right output [1]: [d_date_sk#62] +Arguments: [cs_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(75) CometProject +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61, d_date_sk#62] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] + +(76) CometHashAggregate +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] + +(77) CometExchange +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#59, i_class_id#60, i_category_id#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(78) CometHashAggregate +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] + +(79) CometFilter +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (80) CometProject -Input [4]: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89, d_date_sk#90] -Arguments: [quantity#91, list_price#92], [cs_quantity#87 AS quantity#91, cs_list_price#88 AS list_price#92] - -(81) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95] -Arguments: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95] +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Arguments: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61], [sales#66, number_sales#67, catalog AS channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] + +(81) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(82) CometFilter +Input [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Condition : isnotnull(ws_item_sk#69) + +(83) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#73] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [1]: [ss_item_sk#73] +Arguments: [ws_item_sk#69], [ss_item_sk#73], LeftSemi, BuildRight + +(85) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] + +(86) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_item_sk#69], [i_item_sk#74], Inner, BuildRight + +(87) CometProject +Input [8]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] + +(88) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#78] + +(89) CometBroadcastHashJoin +Left output [6]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight + +(90) CometProject +Input [7]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, d_date_sk#78] +Arguments: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] + +(91) CometHashAggregate +Input [5]: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [partial_sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), partial_count(1)] + +(92) CometExchange +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Arguments: hashpartitioning(i_brand_id#75, i_class_id#76, i_category_id#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(93) CometHashAggregate +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), count(1)] + +(94) CometFilter +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Condition : (isnotnull(sales#82) AND (cast(sales#82 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(95) CometProject +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Arguments: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77], [sales#82, number_sales#83, web AS channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] + +(96) CometUnion +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] +Child 2 Input [6]: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] + +(97) CometExpand +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] + +(98) CometHashAggregate +Input [7]: [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] + +(99) CometExchange +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(100) CometHashAggregate +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [sum(sales#45), sum(number_sales#46)] + +(101) CometTakeOrderedAndProject +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#85 ASC NULLS FIRST,i_brand_id#86 ASC NULLS FIRST,i_class_id#87 ASC NULLS FIRST,i_category_id#88 ASC NULLS FIRST], output=[channel#85,i_brand_id#86,i_class_id#87,i_category_id#88,sum(sales)#93,sum(number_sales)#94]), [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94], 100, 0, [channel#85 ASC NULLS FIRST, i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST], [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] + +(102) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] -(82) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#96] - -(83) CometBroadcastHashJoin -Left output [3]: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95] -Right output [1]: [d_date_sk#96] -Arguments: [ws_sold_date_sk#95], [d_date_sk#96], Inner, BuildRight - -(84) CometProject -Input [4]: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95, d_date_sk#96] -Arguments: [quantity#97, list_price#98], [ws_quantity#93 AS quantity#97, ws_list_price#94 AS list_price#98] - -(85) CometUnion -Child 0 Input [2]: [quantity#85, list_price#86] -Child 1 Input [2]: [quantity#91, list_price#92] -Child 2 Input [2]: [quantity#97, list_price#98] +===== Subqueries ===== -(86) CometHashAggregate -Input [2]: [quantity#85, list_price#86] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* CometColumnarToRow (119) ++- CometHashAggregate (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometUnion (115) + :- CometProject (106) + : +- CometBroadcastHashJoin (105) + : :- CometNativeScan parquet spark_catalog.default.store_sales (103) + : +- ReusedExchange (104) + :- CometProject (110) + : +- CometBroadcastHashJoin (109) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (107) + : +- ReusedExchange (108) + +- CometProject (114) + +- CometBroadcastHashJoin (113) + :- CometNativeScan parquet spark_catalog.default.web_sales (111) + +- ReusedExchange (112) + + +(103) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#97)] +ReadSchema: struct + +(104) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#98] + +(105) CometBroadcastHashJoin +Left output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] +Right output [1]: [d_date_sk#98] +Arguments: [ss_sold_date_sk#97], [d_date_sk#98], Inner, BuildRight + +(106) CometProject +Input [4]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97, d_date_sk#98] +Arguments: [quantity#99, list_price#100], [ss_quantity#95 AS quantity#99, ss_list_price#96 AS list_price#100] + +(107) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#103)] +ReadSchema: struct + +(108) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#104] + +(109) CometBroadcastHashJoin +Left output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] +Right output [1]: [d_date_sk#104] +Arguments: [cs_sold_date_sk#103], [d_date_sk#104], Inner, BuildRight + +(110) CometProject +Input [4]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103, d_date_sk#104] +Arguments: [quantity#105, list_price#106], [cs_quantity#101 AS quantity#105, cs_list_price#102 AS list_price#106] + +(111) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#109)] +ReadSchema: struct + +(112) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#110] + +(113) CometBroadcastHashJoin +Left output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] +Right output [1]: [d_date_sk#110] +Arguments: [ws_sold_date_sk#109], [d_date_sk#110], Inner, BuildRight + +(114) CometProject +Input [4]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109, d_date_sk#110] +Arguments: [quantity#111, list_price#112], [ws_quantity#107 AS quantity#111, ws_list_price#108 AS list_price#112] + +(115) CometUnion +Child 0 Input [2]: [quantity#99, list_price#100] +Child 1 Input [2]: [quantity#105, list_price#106] +Child 2 Input [2]: [quantity#111, list_price#112] + +(116) CometHashAggregate +Input [2]: [quantity#99, list_price#100] Keys: [] -Functions [1]: [partial_avg((cast(quantity#85 as decimal(10,0)) * list_price#86))] +Functions [1]: [partial_avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] -(87) CometExchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(117) CometExchange +Input [2]: [sum#113, count#114] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(88) CometHashAggregate -Input [2]: [sum#99, count#100] +(118) CometHashAggregate +Input [2]: [sum#113, count#114] Keys: [] -Functions [1]: [avg((cast(quantity#85 as decimal(10,0)) * list_price#86))] +Functions [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] -(89) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#101] +(119) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#115] -Subquery:2 Hosting operator id = 60 Hosting Expression = ReusedSubquery Subquery scalar-subquery#43, [id=#44] +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] -Subquery:3 Hosting operator id = 64 Hosting Expression = ReusedSubquery Subquery scalar-subquery#43, [id=#44] +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/extended.txt new file mode 100644 index 0000000000..302366bdb6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/extended.txt @@ -0,0 +1,410 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometUnion + :- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 403 out of 406 eligible operators (99%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/simplified.txt index 12407d32a3..208935d5f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/simplified.txt @@ -1,49 +1,49 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),spark_grouping_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),sum(sales),sum(number_salesL)] CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum,sales,number_sales] + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #12 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #13 + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #9 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #9 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #9 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #2 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -54,43 +54,73 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #9 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #9 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #7 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #11 + CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #15 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/explain.txt index 5d8a967ff0..c57da3c3e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/explain.txt @@ -1,25 +1,25 @@ == Physical Plan == -* ColumnarToRow (77) -+- CometTakeOrderedAndProject (76) - +- CometBroadcastHashJoin (75) - :- CometFilter (56) - : +- CometHashAggregate (55) - : +- CometExchange (54) - : +- CometHashAggregate (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) @@ -28,99 +28,128 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - +- CometBroadcastExchange (74) - +- CometFilter (73) - +- CometHashAggregate (72) - +- CometExchange (71) - +- CometHashAggregate (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometBroadcastHashJoin (60) - : : :- CometFilter (58) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (57) - : : +- ReusedExchange (59) - : +- ReusedExchange (61) - +- CometBroadcastExchange (67) - +- CometProject (66) - +- CometFilter (65) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (64) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometNativeScan: `spark_catalog`.`default`.`item` +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometNativeScan: `spark_catalog`.`default`.`item` +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] @@ -139,9 +168,12 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [d_date_sk#21, d_year#22] @@ -212,331 +244,396 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(34) CometBroadcastHashJoin +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(35) CometBroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [brand_id#24, class_id#25, category_id#26] -(36) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Right output [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(37) CometProject +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] -(39) CometBroadcastHashJoin +(47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(48) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : (((isnotnull(i_item_sk#31) AND isnotnull(i_brand_id#32)) AND isnotnull(i_class_id#33)) AND isnotnull(i_category_id#34)) +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(45) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35, d_week_seq#36] +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#39, d_week_seq#40] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(48) CometFilter -Input [2]: [d_date_sk#35, d_week_seq#36] -Condition : ((isnotnull(d_week_seq#36) AND (d_week_seq#36 = Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#35)) +(56) CometFilter +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) -(49) CometProject -Input [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35], [d_date_sk#35] +(57) CometProject +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -(56) CometFilter -Input [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Condition : (isnotnull(sales#43) AND (cast(sales#43 as decimal(32,6)) > cast(Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(57) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Arguments: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] +(65) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct -(58) CometFilter -Input [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Condition : isnotnull(ss_item_sk#47) +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) -(59) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#51] +(67) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#55] -(60) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [1]: [ss_item_sk#51] -Arguments: [ss_item_sk#47], [ss_item_sk#51], LeftSemi, BuildRight +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight -(61) ReusedExchange [Reuses operator id: 44] -Output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] -(62) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_item_sk#47], [i_item_sk#52], Inner, BuildRight +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight -(63) CometProject -Input [8]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] -(64) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56, d_week_seq#57] +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#60, d_week_seq#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(65) CometFilter -Input [2]: [d_date_sk#56, d_week_seq#57] -Condition : ((isnotnull(d_week_seq#57) AND (d_week_seq#57 = Subquery scalar-subquery#58, [id=#59])) AND isnotnull(d_date_sk#56)) +(73) CometFilter +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) -(66) CometProject -Input [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56], [d_date_sk#56] +(74) CometProject +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] -(67) CometBroadcastExchange -Input [1]: [d_date_sk#56] -Arguments: [d_date_sk#56] +(75) CometBroadcastExchange +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] -(68) CometBroadcastHashJoin -Left output [6]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] -Right output [1]: [d_date_sk#56] -Arguments: [ss_sold_date_sk#50], [d_date_sk#56], Inner, BuildRight +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight -(69) CometProject -Input [7]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55, d_date_sk#56] -Arguments: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] -(70) CometHashAggregate -Input [5]: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [partial_sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), partial_count(1)] +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] -(71) CometExchange -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Arguments: hashpartitioning(i_brand_id#53, i_class_id#54, i_category_id#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(79) CometExchange +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(72) CometHashAggregate -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), count(1)] +(80) CometHashAggregate +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] -(73) CometFilter -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) +(81) CometFilter +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(74) CometBroadcastExchange -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] +(82) CometBroadcastExchange +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -(75) CometBroadcastHashJoin -Left output [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Right output [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [i_brand_id#32, i_class_id#33, i_category_id#34], [i_brand_id#53, i_class_id#54, i_category_id#55], Inner, BuildRight +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight -(76) CometTakeOrderedAndProject -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#32 ASC NULLS FIRST,i_class_id#33 ASC NULLS FIRST,i_category_id#34 ASC NULLS FIRST], output=[channel#42,i_brand_id#32,i_class_id#33,i_category_id#34,sales#43,number_sales#44,channel#63,i_brand_id#53,i_class_id#54,i_category_id#55,sales#64,number_sales#65]), [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65], 100, [i_brand_id#32 ASC NULLS FIRST, i_class_id#33 ASC NULLS FIRST, i_category_id#34 ASC NULLS FIRST], [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -(77) ColumnarToRow [codegen id : 1] -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* ColumnarToRow (94) -+- CometHashAggregate (93) - +- CometExchange (92) - +- CometHashAggregate (91) - +- CometUnion (90) - :- CometProject (81) - : +- CometBroadcastHashJoin (80) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (78) - : +- ReusedExchange (79) - :- CometProject (85) - : +- CometBroadcastHashJoin (84) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (82) - : +- ReusedExchange (83) - +- CometProject (89) - +- CometBroadcastHashJoin (88) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (86) - +- ReusedExchange (87) - - -(78) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Arguments: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] - -(79) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#69] - -(80) CometBroadcastHashJoin -Left output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Right output [1]: [d_date_sk#69] -Arguments: [ss_sold_date_sk#68], [d_date_sk#69], Inner, BuildRight - -(81) CometProject -Input [4]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68, d_date_sk#69] -Arguments: [quantity#70, list_price#71], [ss_quantity#66 AS quantity#70, ss_list_price#67 AS list_price#71] - -(82) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Arguments: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] - -(83) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#75] - -(84) CometBroadcastHashJoin -Left output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Right output [1]: [d_date_sk#75] -Arguments: [cs_sold_date_sk#74], [d_date_sk#75], Inner, BuildRight - -(85) CometProject -Input [4]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74, d_date_sk#75] -Arguments: [quantity#76, list_price#77], [cs_quantity#72 AS quantity#76, cs_list_price#73 AS list_price#77] - -(86) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Arguments: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometNativeScan parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometNativeScan parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] +ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#73] (88) CometBroadcastHashJoin -Left output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#80], [d_date_sk#81], Inner, BuildRight +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight (89) CometProject -Input [4]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80, d_date_sk#81] -Arguments: [quantity#82, list_price#83], [ws_quantity#78 AS quantity#82, ws_list_price#79 AS list_price#83] +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] + +(90) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] +ReadSchema: struct + +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#79] + +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(93) CometProject +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] + +(94) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] +ReadSchema: struct + +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#85] + +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight -(90) CometUnion -Child 0 Input [2]: [quantity#70, list_price#71] -Child 1 Input [2]: [quantity#76, list_price#77] -Child 2 Input [2]: [quantity#82, list_price#83] +(97) CometProject +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] + +(98) CometUnion +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] -(91) CometHashAggregate -Input [2]: [quantity#70, list_price#71] +(99) CometHashAggregate +Input [2]: [quantity#74, list_price#75] Keys: [] -Functions [1]: [partial_avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] -(92) CometExchange -Input [2]: [sum#84, count#85] +(100) CometExchange +Input [2]: [sum#88, count#89] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(93) CometHashAggregate -Input [2]: [sum#84, count#85] +(101) CometHashAggregate +Input [2]: [sum#88, count#89] Keys: [] -Functions [1]: [avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] -(94) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#86] +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#90] -Subquery:2 Hosting operator id = 48 Hosting Expression = Subquery scalar-subquery#37, [id=#38] -* ColumnarToRow (98) -+- CometProject (97) - +- CometFilter (96) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (95) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(95) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] +(103) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] +ReadSchema: struct -(96) CometFilter -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Condition : (((((isnotnull(d_year#88) AND isnotnull(d_moy#89)) AND isnotnull(d_dom#90)) AND (d_year#88 = 2000)) AND (d_moy#89 = 12)) AND (d_dom#90 = 11)) +(104) CometFilter +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 2000)) AND (d_moy#93 = 12)) AND (d_dom#94 = 11)) -(97) CometProject -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87], [d_week_seq#87] +(105) CometProject +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] -(98) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#87] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#91] -Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#58, [id=#59] -* ColumnarToRow (102) -+- CometProject (101) - +- CometFilter (100) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (99) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (107) -(99) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +(107) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] +ReadSchema: struct -(100) CometFilter -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1999)) AND (d_moy#93 = 12)) AND (d_dom#94 = 11)) +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1999)) AND (d_moy#97 = 12)) AND (d_dom#98 = 11)) -(101) CometProject -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91], [d_week_seq#91] +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] -(102) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#91] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/extended.txt new file mode 100644 index 0000000000..fc08251119 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/extended.txt @@ -0,0 +1,290 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometBroadcastHashJoin + :- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : +- Subquery + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 280 out of 284 eligible operators (98%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/simplified.txt index d5143df8c8..92429c0948 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/simplified.txt @@ -1,44 +1,44 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #2 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #11 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #8 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -49,66 +49,74 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #8 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #6 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #2 - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #12 + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #13 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [ss_item_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] Subquery #3 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/explain.txt index 9bdb602daf..40c3fe1b9e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/explain.txt @@ -1,40 +1,48 @@ == Physical Plan == -* ColumnarToRow (23) -+- CometTakeOrderedAndProject (22) - +- CometHashAggregate (21) - +- CometExchange (20) - +- CometHashAggregate (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - +- CometBroadcastExchange (16) - +- CometProject (15) - +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.customer (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + +- CometBroadcastExchange (17) + +- CometProject (16) + +- CometFilter (15) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`customer` +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [c_customer_sk#4, c_current_addr_sk#5] @@ -53,70 +61,80 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Condition : isnotnull(ca_address_sk#6) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(11) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -Right output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight - -(12) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#7, 2, true, false, true) AS ca_state#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#8, 10, true, false, true) AS ca_zip#10] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9, d_year#10, d_qoy#11] +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] -(14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) - -(15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(17) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight - -(18) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] -Arguments: [cs_sales_price#2, ca_zip#8], [cs_sales_price#2, ca_zip#8] - -(19) CometHashAggregate -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] +(12) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight + +(13) CometProject +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) + +(16) CometProject +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] + +(18) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight + +(19) CometProject +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] + +(20) CometHashAggregate +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -(20) CometExchange -Input [2]: [ca_zip#8, sum#12] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) CometHashAggregate -Input [2]: [ca_zip#8, sum#12] -Keys [1]: [ca_zip#8] +(22) CometHashAggregate +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -(22) CometTakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#13] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#8 ASC NULLS FIRST], output=[ca_zip#8,sum(cs_sales_price)#13]), [ca_zip#8, sum(cs_sales_price)#13], 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#13] +(23) CometTakeOrderedAndProject +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#10 ASC NULLS FIRST], output=[ca_zip#10,sum(cs_sales_price)#15]), [ca_zip#10, sum(cs_sales_price)#15], 100, 0, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#15] -(23) ColumnarToRow [codegen id : 1] -Input [2]: [ca_zip#8, sum(cs_sales_price)#13] +(24) CometColumnarToRow [codegen id : 1] +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/extended.txt new file mode 100644 index 0000000000..91516fb7af --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/extended.txt @@ -0,0 +1,26 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 23 out of 23 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/simplified.txt index df01bd6848..ba08c3f0c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - CometHashAggregate [ca_zip,sum(cs_sales_price),sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [ca_zip,sum(cs_sales_price),sum(UnscaledValue(cs_sales_price))] CometExchange [ca_zip] #1 - CometHashAggregate [ca_zip,sum,cs_sales_price] + CometHashAggregate [cs_sales_price] [ca_zip,sum] CometProject [cs_sales_price,ca_zip] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] @@ -12,14 +12,15 @@ WholeStageCodegen (1) CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 - CometFilter [ca_address_sk,ca_state,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_zip] + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] + CometFilter [ca_address_sk,ca_state,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/explain.txt index 3fd94211dd..2a3c8932ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/explain.txt @@ -1,49 +1,53 @@ == Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`call_center` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometNativeScan parquet spark_catalog.default.call_center (29) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] @@ -61,9 +65,11 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(6) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] -Arguments: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +ReadSchema: struct (7) CometProject Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] @@ -86,9 +92,11 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(12) CometNativeScan parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] -Arguments: [cr_order_number#12, cr_returned_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +ReadSchema: struct (13) CometProject Input [2]: [cr_order_number#12, cr_returned_date_sk#13] @@ -107,9 +115,12 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct (18) CometFilter Input [2]: [d_date_sk#14, d_date#15] @@ -132,13 +143,16 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#16)) (25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] @@ -157,9 +171,12 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometNativeScan: `spark_catalog`.`default`.`call_center` +(29) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] -Arguments: [cc_call_center_sk#18, cc_county#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)] +ReadSchema: struct (30) CometFilter Input [2]: [cc_call_center_sk#18, cc_county#19] @@ -187,7 +204,7 @@ Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Keys [1]: [cs_order_number#5] Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] -(36) ColumnarToRow [codegen id : 1] +(36) CometColumnarToRow [codegen id : 1] Input [3]: [cs_order_number#5, sum#20, sum#21] (37) HashAggregate [codegen id : 1] @@ -204,14 +221,15 @@ Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] Results [3]: [sum#20, sum#21, count#25] -(39) Exchange +(39) CometColumnarExchange Input [3]: [sum#20, sum#21, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 2] +(40) CometHashAggregate Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#23,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/extended.txt new file mode 100644 index 0000000000..9889eeaca3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/simplified.txt index 8a0dd1344a..429d83d08c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/simplified.txt @@ -1,44 +1,45 @@ WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [cs_order_number,sum,sum,cs_ext_ship_cost,cs_net_profit] - CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] - CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] - CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometSort [cs_warehouse_sk,cs_order_number] - CometExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - CometSort [cr_order_number] - CometExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_order_number,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [cc_call_center_sk] #7 - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_county] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(cs_order_number),sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [cs_ext_ship_cost,cs_net_profit] [cs_order_number,sum,sum] + CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] + CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] + CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cc_call_center_sk] #7 + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/explain.txt index 8d26f05153..0e4ffc8864 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (42) -+- CometTakeOrderedAndProject (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) : :- CometProject (27) : : +- CometBroadcastHashJoin (26) : : :- CometProject (24) @@ -19,41 +19,51 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`item` (33) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.item (34) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] @@ -72,9 +82,13 @@ Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] @@ -93,13 +107,16 @@ Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_quarter_name#17] -Arguments: [d_date_sk#16, d_quarter_name#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#17, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#16)) (15) CometProject Input [2]: [d_date_sk#16, d_quarter_name#17] @@ -118,13 +135,16 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_quarter_name#19] -Arguments: [d_date_sk#18, d_quarter_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct (20) CometFilter Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#19, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) (21) CometProject Input [2]: [d_date_sk#18, d_quarter_name#19] @@ -155,66 +175,80 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(28) CometNativeScan: `spark_catalog`.`default`.`store` +(28) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (29) CometFilter Input [2]: [s_store_sk#21, s_state#22] Condition : isnotnull(s_store_sk#21) -(30) CometBroadcastExchange +(30) CometProject Input [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#22, 2, true, false, true) AS s_state#23] + +(31) CometBroadcastExchange +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] -(31) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [2]: [s_store_sk#21, s_state#22] +Right output [2]: [s_store_sk#21, s_state#23] Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight -(32) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] +(33) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] -(33) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(34) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(34) CometFilter -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) +(35) CometFilter +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) -(35) CometBroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(36) CometProject +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#25, 16, true, false, true) AS i_item_id#27, i_item_desc#26] -(36) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Right output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_item_sk#1], [i_item_sk#23], Inner, BuildRight +(37) CometBroadcastExchange +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] -(37) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] +(38) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight -(38) CometHashAggregate -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(39) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] + +(40) CometHashAggregate +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] -(39) CometExchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(41) CometExchange +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(40) CometHashAggregate -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(42) CometHashAggregate +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] -(41) CometTakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#24 ASC NULLS FIRST,i_item_desc#25 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#24,i_item_desc#25,s_state#22,store_sales_quantitycount#44,store_sales_quantityave#45,store_sales_quantitystdev#46,store_sales_quantitycov#47,as_store_returns_quantitycount#48,as_store_returns_quantityave#49,as_store_returns_quantitystdev#50,store_returns_quantitycov#51,catalog_sales_quantitycount#52,catalog_sales_quantityave#53,catalog_sales_quantitystdev#54,catalog_sales_quantitycov#55]), [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55], 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] +(43) CometTakeOrderedAndProject +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] -(42) ColumnarToRow [codegen id : 1] -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] +(44) CometColumnarToRow [codegen id : 1] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/simplified.txt index 746826546e..97c37ed5a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - CometHashAggregate [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] + CometHashAggregate [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] CometExchange [i_item_id,i_item_desc,s_state] #1 - CometHashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,ss_quantity,sr_return_quantity,cs_quantity] + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] @@ -20,25 +20,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] ReusedExchange [d_date_sk] #5 CometBroadcastExchange [s_store_sk,s_state] #6 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/explain.txt index c4203a3d92..c33d4125b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* ColumnarToRow (41) -+- CometTakeOrderedAndProject (40) - +- CometHashAggregate (39) - +- CometExchange (38) - +- CometHashAggregate (37) - +- CometExpand (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (43) ++- CometTakeOrderedAndProject (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometExpand (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) : : :- CometProject (19) : : : +- CometBroadcastHashJoin (18) : : : :- CometProject (14) @@ -18,45 +18,54 @@ : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (15) - : : +- CometBroadcastExchange (22) - : : +- CometFilter (21) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - : +- CometBroadcastExchange (28) - : +- CometProject (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - +- CometBroadcastExchange (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`item` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.date_dim (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.item (32) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Arguments: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#10)) (5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] @@ -75,9 +84,12 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(9) CometNativeScan: `spark_catalog`.`default`.`customer` +(9) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (10) CometFilter Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] @@ -100,9 +112,12 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(15) CometNativeScan parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [1]: [cd_demo_sk#19] @@ -121,95 +136,112 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) -(22) CometBroadcastExchange +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24, d_year#25] - -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) - -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] - -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight - -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(26) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) + +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight + +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] + +(38) CometExpand +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] -(31) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] - -(36) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] - -(37) CometHashAggregate -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(39) CometHashAggregate +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] -(38) CometExchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(40) CometExchange +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) CometHashAggregate -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(41) CometHashAggregate +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] -(40) CometTakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#29 ASC NULLS FIRST,ca_state#30 ASC NULLS FIRST,ca_county#31 ASC NULLS FIRST,i_item_id#28 ASC NULLS FIRST], output=[i_item_id#28,ca_country#29,ca_state#30,ca_county#31,agg1#47,agg2#48,agg3#49,agg4#50,agg5#51,agg6#52,agg7#53]), [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53], 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] +(42) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] -(41) ColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] +(43) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/extended.txt new file mode 100644 index 0000000000..63cf56de78 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/simplified.txt index e795eab5b8..4507330941 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] + CometHashAggregate [spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] @@ -19,25 +19,27 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state,ca_country] + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/explain.txt index eb29ad43d0..78e43d0122 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometTakeOrderedAndProject (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (36) ++- CometTakeOrderedAndProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (19) : : +- CometBroadcastHashJoin (18) : : :- CometProject (14) @@ -16,26 +16,33 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + : : : +- CometNativeScan parquet spark_catalog.default.item (9) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - +- ReusedExchange (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` + : : +- CometNativeScan parquet spark_catalog.default.customer (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.store (26) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -45,9 +52,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] @@ -66,9 +77,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] @@ -76,93 +90,119 @@ Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnul (11) CometProject Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [i_item_sk#9, i_brand_id#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#11, 50, true, false, true) AS i_brand#15, i_manufact_id#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#13, 50, true, false, true) AS i_manufact#16] (12) CometBroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(15) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] +(15) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (16) CometFilter -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) (17) CometBroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [c_customer_sk#17, c_current_addr_sk#18] (18) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Right output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_customer_sk#5], [c_customer_sk#15], Inner, BuildRight +Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight (19) CometProject -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] +Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] +(20) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#19, ca_zip#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (21) CometFilter -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) - -(22) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Right output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight - -(24) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] - -(25) ReusedExchange [Reuses operator id: 22] -Output [2]: [s_store_sk#19, s_zip#20] - -(26) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Right output [2]: [s_store_sk#19, s_zip#20] -Arguments: [ss_store_sk#6], [s_store_sk#19], Inner, NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)), BuildRight - -(27) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] -Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(28) CometHashAggregate -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +Input [2]: [ca_address_sk#19, ca_zip#20] +Condition : (isnotnull(ca_address_sk#19) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true))) + +(22) CometProject +Input [2]: [ca_address_sk#19, ca_zip#20] +Arguments: [ca_address_sk#19, ca_zip#21], [ca_address_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true) AS ca_zip#21] + +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [ca_address_sk#19, ca_zip#21] + +(24) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] +Right output [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight + +(25) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] + +(26) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#22, s_zip#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [s_store_sk#22, s_zip#23] +Condition : (isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true)) AND isnotnull(s_store_sk#22)) + +(28) CometProject +Input [2]: [s_store_sk#22, s_zip#23] +Arguments: [s_store_sk#22, s_zip#24], [s_store_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true) AS s_zip#24] + +(29) CometBroadcastExchange +Input [2]: [s_store_sk#22, s_zip#24] +Arguments: [s_store_sk#22, s_zip#24] + +(30) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] +Right output [2]: [s_store_sk#22, s_zip#24] +Arguments: [ss_store_sk#6], [s_store_sk#22], Inner, NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5)), BuildRight + +(31) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21, s_store_sk#22, s_zip#24] +Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] + +(32) CometHashAggregate +Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -(29) CometExchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(33) CometExchange +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Arguments: hashpartitioning(i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(30) CometHashAggregate -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(34) CometHashAggregate +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] -(31) CometTakeOrderedAndProject -Input [5]: [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#24 DESC NULLS LAST,brand#23 ASC NULLS FIRST,brand_id#22 ASC NULLS FIRST,i_manufact_id#12 ASC NULLS FIRST,i_manufact#13 ASC NULLS FIRST], output=[brand_id#22,brand#23,i_manufact_id#12,i_manufact#13,ext_price#24]), [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24], 100, [ext_price#24 DESC NULLS LAST, brand#23 ASC NULLS FIRST, brand_id#22 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] +(35) CometTakeOrderedAndProject +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#28 DESC NULLS LAST,brand#27 ASC NULLS FIRST,brand_id#26 ASC NULLS FIRST,i_manufact_id#12 ASC NULLS FIRST,i_manufact#16 ASC NULLS FIRST], output=[brand_id#26,brand#27,i_manufact_id#12,i_manufact#16,ext_price#28]), [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28], 100, 0, [ext_price#28 DESC NULLS LAST, brand#27 ASC NULLS FIRST, brand_id#26 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#16 ASC NULLS FIRST], [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] -(32) ColumnarToRow [codegen id : 1] -Input [5]: [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] +(36) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/extended.txt new file mode 100644 index 0000000000..fbe6798e3d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/extended.txt @@ -0,0 +1,38 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 35 out of 35 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/simplified.txt index 92be42b721..675500cd0b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [brand_id,brand,i_manufact_id,i_manufact,ext_price] - CometHashAggregate [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - CometHashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] @@ -17,18 +17,22 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 - CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_zip] #5 - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] - ReusedExchange [s_store_sk,s_zip] #5 + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [s_store_sk,s_zip] #6 + CometProject [s_zip] [s_store_sk,s_zip] + CometFilter [s_store_sk,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/explain.txt index 93dc1af246..40e213818e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/explain.txt @@ -1,50 +1,57 @@ == Physical Plan == -* ColumnarToRow (33) -+- CometSort (32) - +- CometColumnarExchange (31) - +- CometProject (30) - +- CometBroadcastHashJoin (29) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometHashAggregate (13) - : : +- CometExchange (12) - : : +- CometHashAggregate (11) - : : +- CometProject (10) - : : +- CometBroadcastHashJoin (9) +* CometColumnarToRow (34) ++- CometSort (33) + +- CometExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometHashAggregate (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) : : :- CometUnion (5) : : : :- CometProject (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometProject (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : +- CometBroadcastExchange (8) - : : +- CometFilter (7) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (6) - : +- CometBroadcastExchange (17) - : +- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - +- CometBroadcastExchange (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometHashAggregate (21) - : +- ReusedExchange (20) - +- CometBroadcastExchange (25) - +- CometProject (24) - +- CometFilter (23) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (3) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (6) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.date_dim (15) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometHashAggregate (22) + : +- ReusedExchange (21) + +- CometBroadcastExchange (26) + +- CometProject (25) + +- CometFilter (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (23) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] -Arguments: [ws_ext_sales_price#1, ws_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#2)] +ReadSchema: struct (2) CometProject Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(3) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] -Arguments: [cs_ext_sales_price#5, cs_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#6)] +ReadSchema: struct (4) CometProject Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] @@ -54,120 +61,133 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(6) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] +ReadSchema: struct (7) CometFilter Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(8) CometBroadcastExchange +(8) CometProject Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12], [d_date_sk#9, d_week_seq#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#11, 9, true, false, true) AS d_day_name#12] -(9) CometBroadcastHashJoin +(9) CometBroadcastExchange +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12] + +(10) CometBroadcastHashJoin Left output [2]: [sold_date_sk#3, sales_price#4] -Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(10) CometProject -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [sales_price#4, d_week_seq#10, d_day_name#11], [sales_price#4, d_week_seq#10, d_day_name#11] +(11) CometProject +Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sales_price#4, d_week_seq#10, d_day_name#12], [sales_price#4, d_week_seq#10, d_day_name#12] -(11) CometHashAggregate -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] +(12) CometHashAggregate +Input [3]: [sales_price#4, d_week_seq#10, d_day_name#12] Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(12) CometExchange -Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] +(13) CometExchange +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) CometHashAggregate -Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] +(14) CometHashAggregate +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_week_seq#19, d_year#20] -Arguments: [d_week_seq#19, d_year#20] +(15) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#20, d_year#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] +ReadSchema: struct -(15) CometFilter -Input [2]: [d_week_seq#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_week_seq#19)) +(16) CometFilter +Input [2]: [d_week_seq#20, d_year#21] +Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_week_seq#20)) -(16) CometProject -Input [2]: [d_week_seq#19, d_year#20] -Arguments: [d_week_seq#19], [d_week_seq#19] +(17) CometProject +Input [2]: [d_week_seq#20, d_year#21] +Arguments: [d_week_seq#20], [d_week_seq#20] -(17) CometBroadcastExchange -Input [1]: [d_week_seq#19] -Arguments: [d_week_seq#19] +(18) CometBroadcastExchange +Input [1]: [d_week_seq#20] +Arguments: [d_week_seq#20] -(18) CometBroadcastHashJoin -Left output [8]: [d_week_seq#10, sun_sales#21, mon_sales#22, tue_sales#23, wed_sales#24, thu_sales#25, fri_sales#26, sat_sales#27] -Right output [1]: [d_week_seq#19] -Arguments: [d_week_seq#10], [d_week_seq#19], Inner, BuildRight +(19) CometBroadcastHashJoin +Left output [8]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28] +Right output [1]: [d_week_seq#20] +Arguments: [d_week_seq#10], [d_week_seq#20], Inner, BuildRight -(19) CometProject -Input [9]: [d_week_seq#10, sun_sales#21, mon_sales#22, tue_sales#23, wed_sales#24, thu_sales#25, fri_sales#26, sat_sales#27, d_week_seq#19] -Arguments: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35], [d_week_seq#10 AS d_week_seq1#28, sun_sales#21 AS sun_sales1#29, mon_sales#22 AS mon_sales1#30, tue_sales#23 AS tue_sales1#31, wed_sales#24 AS wed_sales1#32, thu_sales#25 AS thu_sales1#33, fri_sales#26 AS fri_sales1#34, sat_sales#27 AS sat_sales1#35] +(20) CometProject +Input [9]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, d_week_seq#20] +Arguments: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36], [d_week_seq#10 AS d_week_seq1#29, sun_sales#22 AS sun_sales1#30, mon_sales#23 AS mon_sales1#31, tue_sales#24 AS tue_sales1#32, wed_sales#25 AS wed_sales1#33, thu_sales#26 AS thu_sales1#34, fri_sales#27 AS fri_sales1#35, sat_sales#28 AS sat_sales1#36] -(20) ReusedExchange [Reuses operator id: 12] -Output [8]: [d_week_seq#36, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] +(21) ReusedExchange [Reuses operator id: 13] +Output [8]: [d_week_seq#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] -(21) CometHashAggregate -Input [8]: [d_week_seq#36, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] -Keys [1]: [d_week_seq#36] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#44 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Saturday ) THEN sales_price#4 END))] +(22) CometHashAggregate +Input [8]: [d_week_seq#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] +Keys [1]: [d_week_seq#37] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(23) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#45, d_year#46] -Arguments: [d_week_seq#45, d_year#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] +ReadSchema: struct -(23) CometFilter +(24) CometFilter Input [2]: [d_week_seq#45, d_year#46] Condition : ((isnotnull(d_year#46) AND (d_year#46 = 2002)) AND isnotnull(d_week_seq#45)) -(24) CometProject +(25) CometProject Input [2]: [d_week_seq#45, d_year#46] Arguments: [d_week_seq#45], [d_week_seq#45] -(25) CometBroadcastExchange +(26) CometBroadcastExchange Input [1]: [d_week_seq#45] Arguments: [d_week_seq#45] -(26) CometBroadcastHashJoin -Left output [8]: [d_week_seq#36, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53] +(27) CometBroadcastHashJoin +Left output [8]: [d_week_seq#37, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53] Right output [1]: [d_week_seq#45] -Arguments: [d_week_seq#36], [d_week_seq#45], Inner, BuildRight +Arguments: [d_week_seq#37], [d_week_seq#45], Inner, BuildRight -(27) CometProject -Input [9]: [d_week_seq#36, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53, d_week_seq#45] -Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61], [d_week_seq#36 AS d_week_seq2#54, sun_sales#47 AS sun_sales2#55, mon_sales#48 AS mon_sales2#56, tue_sales#49 AS tue_sales2#57, wed_sales#50 AS wed_sales2#58, thu_sales#51 AS thu_sales2#59, fri_sales#52 AS fri_sales2#60, sat_sales#53 AS sat_sales2#61] +(28) CometProject +Input [9]: [d_week_seq#37, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53, d_week_seq#45] +Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61], [d_week_seq#37 AS d_week_seq2#54, sun_sales#47 AS sun_sales2#55, mon_sales#48 AS mon_sales2#56, tue_sales#49 AS tue_sales2#57, wed_sales#50 AS wed_sales2#58, thu_sales#51 AS thu_sales2#59, fri_sales#52 AS fri_sales2#60, sat_sales#53 AS sat_sales2#61] -(28) CometBroadcastExchange +(29) CometBroadcastExchange Input [8]: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -(29) CometBroadcastHashJoin -Left output [8]: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35] +(30) CometBroadcastHashJoin +Left output [8]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36] Right output [8]: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -Arguments: [d_week_seq1#28], [(d_week_seq2#54 - 53)], Inner, BuildRight +Arguments: [d_week_seq1#29], [(d_week_seq2#54 - 53)], Inner, BuildRight -(30) CometProject -Input [16]: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35, d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -Arguments: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#28, round((sun_sales1#29 / sun_sales2#55), 2) AS round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1#30 / mon_sales2#56), 2) AS round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1#31 / tue_sales2#57), 2) AS round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1#32 / wed_sales2#58), 2) AS round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1#33 / thu_sales2#59), 2) AS round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1#34 / fri_sales2#60), 2) AS round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1#35 / sat_sales2#61), 2) AS round((sat_sales1 / sat_sales2), 2)#68] +(31) CometProject +Input [16]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36, d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#29, round((sun_sales1#30 / sun_sales2#55), 2) AS round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1#31 / mon_sales2#56), 2) AS round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1#32 / tue_sales2#57), 2) AS round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1#33 / wed_sales2#58), 2) AS round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1#34 / thu_sales2#59), 2) AS round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1#35 / fri_sales2#60), 2) AS round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1#36 / sat_sales2#61), 2) AS round((sat_sales1 / sat_sales2), 2)#68] -(31) CometColumnarExchange -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] -Arguments: rangepartitioning(d_week_seq1#28 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(32) CometExchange +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] +Arguments: rangepartitioning(d_week_seq1#29 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(32) CometSort -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] -Arguments: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#28 ASC NULLS FIRST] +(33) CometSort +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#29 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 1] -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] +(34) CometColumnarToRow [codegen id : 1] +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/extended.txt new file mode 100644 index 0000000000..7e6f4d3a3c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/simplified.txt index 8550678605..e4b6e81639 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/simplified.txt @@ -1,35 +1,36 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] - CometColumnarExchange [d_week_seq1] #1 + CometExchange [d_week_seq1] #1 CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] CometBroadcastHashJoin [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] - CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] CometExchange [d_week_seq] #2 - CometHashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum,d_day_name,sales_price] + CometHashAggregate [d_day_name,sales_price] [d_week_seq,sum,sum,sum,sum,sum,sum,sum] CometProject [sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] CometUnion [sold_date_sk,sales_price] CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] - CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 CometBroadcastExchange [d_week_seq] #6 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/explain.txt index 01031df826..684cbba7f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/explain.txt @@ -1,116 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/extended.txt new file mode 100644 index 0000000000..7fd40cf7dd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/simplified.txt index 515af15c2b..0b88804c69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] CometExchange [i_class] #1 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/explain.txt index 4a2b1204d9..68db1e1ee5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (24) +* CometColumnarToRow (24) +- CometTakeOrderedAndProject (23) +- CometFilter (22) +- CometHashAggregate (21) @@ -12,30 +12,37 @@ : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (8) + : +- CometNativeScan parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [w_warehouse_sk#5, w_warehouse_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] @@ -54,9 +61,12 @@ Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -(8) CometNativeScan: `spark_catalog`.`default`.`item` +(8) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8, i_current_price#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] @@ -64,64 +74,67 @@ Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (10) CometProject Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8], [i_item_sk#7, i_item_id#8] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#8, 16, true, false, true) AS i_item_id#10] (11) CometBroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: [i_item_sk#7, i_item_id#8] +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] (12) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -Right output [2]: [i_item_sk#7, i_item_id#8] +Right output [2]: [i_item_sk#7, i_item_id#10] Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_date#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) (16) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] (17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [inv_date_sk#4], [d_date_sk#10], Inner, BuildRight +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight (18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] (19) CometHashAggregate -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (20) CometExchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#12, sum#13] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#12, sum#13] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (22) CometFilter -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] -Condition : (CASE WHEN (inv_before#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#15 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#14 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#15 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#14 as double)))))) <= 1.5) END) +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) (23) CometTakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#8 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#8,inv_before#14,inv_after#15]), [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15], 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] -(24) ColumnarToRow [codegen id : 1] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] +(24) CometColumnarToRow [codegen id : 1] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/extended.txt new file mode 100644 index 0000000000..92628ea98d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/extended.txt @@ -0,0 +1,26 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 23 out of 23 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/simplified.txt index 25f0c2270e..f0cc321417 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] - CometHashAggregate [w_warehouse_name,i_item_id,inv_before,inv_after,sum,sum,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] + CometHashAggregate [sum,sum] [w_warehouse_name,i_item_id,inv_before,inv_after,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] CometExchange [w_warehouse_name,i_item_id] #1 - CometHashAggregate [w_warehouse_name,i_item_id,sum,sum,d_date,inv_quantity_on_hand] + CometHashAggregate [d_date,inv_quantity_on_hand] [w_warehouse_name,i_item_id,sum,sum] CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] @@ -13,14 +13,14 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_id] #3 - CometProject [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #4 CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/explain.txt index 4a89a80d9c..fa4f406442 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/explain.txt @@ -1,41 +1,49 @@ == Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (25) ++- CometTakeOrderedAndProject (24) + +- CometHashAggregate (23) + +- CometExchange (22) + +- CometHashAggregate (21) + +- CometExpand (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.warehouse (15) -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] @@ -58,70 +66,80 @@ Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Condition : isnotnull(i_item_sk#7) -(11) CometBroadcastExchange +(11) CometProject Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(13) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] - -(19) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(20) CometHashAggregate -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(15) CometNativeScan parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) + +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] + +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight + +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] + +(20) CometExpand +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] + +(21) CometHashAggregate +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(21) CometExchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(22) CometExchange +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#18, count#19] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(23) CometHashAggregate +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] -(23) CometTakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#20 ASC NULLS FIRST,i_product_name#13 ASC NULLS FIRST,i_brand#14 ASC NULLS FIRST,i_class#15 ASC NULLS FIRST,i_category#16 ASC NULLS FIRST], output=[i_product_name#13,i_brand#14,i_class#15,i_category#16,qoh#20]), [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20], 100, [qoh#20 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] +(24) CometTakeOrderedAndProject +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] -(24) ColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] +(25) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/extended.txt new file mode 100644 index 0000000000..275cecfd63 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/simplified.txt index 5e6610f3bc..8d2b671164 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,spark_grouping_id,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [spark_grouping_id,sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] @@ -13,14 +13,15 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/explain.txt index a7df911fce..8fc110239e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (67) +* CometColumnarToRow (67) +- CometHashAggregate (66) +- CometExchange (65) +- CometHashAggregate (64) @@ -12,7 +12,7 @@ : : : +- CometExchange (23) : : : +- CometProject (22) : : : +- CometBroadcastHashJoin (21) - : : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (20) : : : +- CometProject (19) : : : +- CometFilter (18) @@ -24,14 +24,14 @@ : : : :- CometProject (9) : : : : +- CometBroadcastHashJoin (8) : : : : :- CometFilter (3) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (2) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : : +- CometBroadcastExchange (12) : : : +- CometFilter (11) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (10) + : : : +- CometNativeScan parquet spark_catalog.default.item (10) : : +- CometSort (38) : : +- CometProject (37) : : +- CometFilter (36) @@ -42,14 +42,14 @@ : : +- CometBroadcastHashJoin (31) : : :- CometProject (27) : : : +- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (25) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (25) : : +- CometBroadcastExchange (30) : : +- CometFilter (29) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (28) + : : +- CometNativeScan parquet spark_catalog.default.customer (28) : +- CometBroadcastExchange (44) : +- CometProject (43) : +- CometFilter (42) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (41) + : +- CometNativeScan parquet spark_catalog.default.date_dim (41) +- CometProject (62) +- CometBroadcastHashJoin (61) :- CometProject (59) @@ -58,7 +58,7 @@ : : +- CometExchange (51) : : +- CometProject (50) : : +- CometBroadcastHashJoin (49) - : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (47) + : : :- CometNativeScan parquet spark_catalog.default.web_sales (47) : : +- ReusedExchange (48) : +- CometSort (57) : +- CometProject (56) @@ -68,21 +68,31 @@ +- ReusedExchange (60) -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +ReadSchema: struct -(2) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(2) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (3) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] @@ -105,9 +115,12 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(10) CometNativeScan: `spark_catalog`.`default`.`item` +(10) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (11) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] @@ -169,9 +182,12 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(25) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(25) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (26) CometFilter Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] @@ -181,9 +197,12 @@ Condition : isnotnull(ss_customer_sk#17) Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(28) CometNativeScan: `spark_catalog`.`default`.`customer` +(28) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (29) CometFilter Input [1]: [c_customer_sk#21] @@ -237,9 +256,12 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(41) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27, d_year#28, d_moy#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct (42) CometFilter Input [3]: [d_date_sk#27, d_year#28, d_moy#29] @@ -262,9 +284,12 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] -(47) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(47) CometNativeScan parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Arguments: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] +ReadSchema: struct (48) ReusedExchange [Reuses operator id: 20] Output [1]: [item_sk#36] @@ -345,13 +370,13 @@ Input [2]: [sum#45, isEmpty#46] Keys: [] Functions [1]: [sum(sales#30)] -(67) ColumnarToRow [codegen id : 1] +(67) CometColumnarToRow [codegen id : 1] Input [1]: [sum(sales)#47] ===== Subqueries ===== Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] -* ColumnarToRow (85) +* CometColumnarToRow (85) +- CometHashAggregate (84) +- CometExchange (83) +- CometHashAggregate (82) @@ -363,17 +388,21 @@ Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquer :- CometProject (72) : +- CometBroadcastHashJoin (71) : :- CometFilter (69) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (68) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) : +- ReusedExchange (70) +- CometBroadcastExchange (76) +- CometProject (75) +- CometFilter (74) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) -(68) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(68) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] -Arguments: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#51)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (69) CometFilter Input [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] @@ -391,9 +420,12 @@ Arguments: [ss_customer_sk#48], [c_customer_sk#52], Inner, BuildRight Input [5]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] Arguments: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] -(73) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(73) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#53, d_year#54] -Arguments: [d_date_sk#53, d_year#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct (74) CometFilter Input [2]: [d_date_sk#53, d_year#54] @@ -444,7 +476,7 @@ Input [1]: [max#58] Keys: [] Functions [1]: [max(csales#57)] -(85) ColumnarToRow [codegen id : 1] +(85) CometColumnarToRow [codegen id : 1] Input [1]: [tpcds_cmax#59] Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/extended.txt new file mode 100644 index 0000000000..a914b98c8e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/extended.txt @@ -0,0 +1,121 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometSort + : +- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 115 out of 117 eligible operators (98%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/simplified.txt index 5159f36c51..3604457d42 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [sum(sales),sum,isEmpty,sum(sales)] + CometHashAggregate [sum,isEmpty] [sum(sales),sum(sales)] CometExchange #1 - CometHashAggregate [sum,isEmpty,sales] + CometHashAggregate [sales] [sum,isEmpty] CometUnion [sales] CometProject [cs_quantity,cs_list_price] [sales] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] @@ -13,11 +13,11 @@ WholeStageCodegen (1) CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] @@ -25,53 +25,53 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [tpcds_cmax,max,max(csales)] + CometHashAggregate [max] [tpcds_cmax,max(csales)] CometExchange #9 - CometHashAggregate [max,csales] - CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #10 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] ReusedExchange [c_customer_sk] #8 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #7 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_quantity,ws_list_price] [sales] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] @@ -80,12 +80,12 @@ WholeStageCodegen (1) CometExchange [ws_bill_customer_sk] #13 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] ReusedExchange [c_customer_sk,sum,isEmpty] #7 ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/explain.txt index 1611bcf325..bacfb6e6c8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/explain.txt @@ -1,21 +1,21 @@ == Physical Plan == -* ColumnarToRow (87) -+- CometTakeOrderedAndProject (86) - +- CometUnion (85) - :- CometHashAggregate (62) - : +- CometExchange (61) - : +- CometHashAggregate (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (53) - : : +- CometBroadcastHashJoin (52) +* CometColumnarToRow (88) ++- CometTakeOrderedAndProject (87) + +- CometUnion (86) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) : : :- CometSortMergeJoin (40) : : : :- CometSort (25) : : : : +- CometExchange (24) : : : : +- CometProject (23) : : : : +- CometBroadcastHashJoin (22) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (21) : : : : +- CometProject (20) : : : : +- CometFilter (19) @@ -27,14 +27,14 @@ : : : : :- CometProject (10) : : : : : +- CometBroadcastHashJoin (9) : : : : : :- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (3) : : : : : +- CometBroadcastExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (5) : : : : +- CometBroadcastExchange (13) : : : : +- CometFilter (12) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) + : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : +- CometSort (39) : : : +- CometProject (38) : : : +- CometFilter (37) @@ -45,68 +45,80 @@ : : : +- CometBroadcastHashJoin (32) : : : :- CometProject (28) : : : : +- CometFilter (27) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) : : : +- CometBroadcastExchange (31) : : : +- CometFilter (30) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - : : +- CometBroadcastExchange (51) - : : +- CometSortMergeJoin (50) - : : :- CometSort (44) - : : : +- CometExchange (43) - : : : +- CometFilter (42) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (41) - : : +- CometSort (49) - : : +- CometProject (48) - : : +- CometFilter (47) - : : +- CometHashAggregate (46) - : : +- ReusedExchange (45) - : +- CometBroadcastExchange (57) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (54) - +- CometHashAggregate (84) - +- CometExchange (83) - +- CometHashAggregate (82) - +- CometProject (81) - +- CometBroadcastHashJoin (80) - :- CometProject (78) - : +- CometBroadcastHashJoin (77) - : :- CometSortMergeJoin (75) - : : :- CometSort (69) - : : : +- CometExchange (68) - : : : +- CometProject (67) - : : : +- CometBroadcastHashJoin (66) - : : : :- CometFilter (64) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (63) - : : : +- ReusedExchange (65) - : : +- CometSort (74) - : : +- CometProject (73) - : : +- CometFilter (72) - : : +- CometHashAggregate (71) - : : +- ReusedExchange (70) - : +- ReusedExchange (76) - +- ReusedExchange (79) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.customer (29) + : : +- CometBroadcastExchange (52) + : : +- CometProject (51) + : : +- CometSortMergeJoin (50) + : : :- CometSort (44) + : : : +- CometExchange (43) + : : : +- CometFilter (42) + : : : +- CometNativeScan parquet spark_catalog.default.customer (41) + : : +- CometSort (49) + : : +- CometProject (48) + : : +- CometFilter (47) + : : +- CometHashAggregate (46) + : : +- ReusedExchange (45) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + +- CometHashAggregate (85) + +- CometExchange (84) + +- CometHashAggregate (83) + +- CometProject (82) + +- CometBroadcastHashJoin (81) + :- CometProject (79) + : +- CometBroadcastHashJoin (78) + : :- CometSortMergeJoin (76) + : : :- CometSort (70) + : : : +- CometExchange (69) + : : : +- CometProject (68) + : : : +- CometBroadcastHashJoin (67) + : : : :- CometFilter (65) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (64) + : : : +- ReusedExchange (66) + : : +- CometSort (75) + : : +- CometProject (74) + : : +- CometFilter (73) + : : +- CometHashAggregate (72) + : : +- ReusedExchange (71) + : +- ReusedExchange (77) + +- ReusedExchange (80) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(5) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct (6) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] @@ -129,9 +141,12 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(11) CometNativeScan: `spark_catalog`.`default`.`item` +(11) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (12) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] @@ -193,9 +208,12 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (27) CometFilter Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] @@ -205,9 +223,12 @@ Condition : isnotnull(ss_customer_sk#17) Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(29) CometNativeScan: `spark_catalog`.`default`.`customer` +(29) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (30) CometFilter Input [1]: [c_customer_sk#21] @@ -257,9 +278,12 @@ Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold Right output [1]: [c_customer_sk#21] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi -(41) CometNativeScan: `spark_catalog`.`default`.`customer` +(41) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (42) CometFilter Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] @@ -298,261 +322,279 @@ Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Right output [1]: [c_customer_sk#21] Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi -(51) CometBroadcastExchange +(51) CometProject Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#31] + +(52) CometBroadcastExchange +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] -(52) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight -(53) CometProject -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29] - -(54) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30, d_year#31, d_moy#32] - -(55) CometFilter -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Condition : ((((isnotnull(d_year#31) AND isnotnull(d_moy#32)) AND (d_year#31 = 2000)) AND (d_moy#32 = 2)) AND isnotnull(d_date_sk#30)) - -(56) CometProject -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30], [d_date_sk#30] - -(57) CometBroadcastExchange -Input [1]: [d_date_sk#30] -Arguments: [d_date_sk#30] - -(58) CometBroadcastHashJoin -Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29] -Right output [1]: [d_date_sk#30] -Arguments: [cs_sold_date_sk#5], [d_date_sk#30], Inner, BuildRight - -(59) CometProject -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29, d_date_sk#30] -Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29], [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29] - -(60) CometHashAggregate -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29] -Keys [2]: [c_last_name#29, c_first_name#28] +(54) CometProject +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] + +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) + +(57) CometProject +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] + +(59) CometBroadcastHashJoin +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight + +(60) CometProject +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] + +(61) CometHashAggregate +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(61) CometExchange -Input [4]: [c_last_name#29, c_first_name#28, sum#33, isEmpty#34] -Arguments: hashpartitioning(c_last_name#29, c_first_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(62) CometExchange +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(62) CometHashAggregate -Input [4]: [c_last_name#29, c_first_name#28, sum#33, isEmpty#34] -Keys [2]: [c_last_name#29, c_first_name#28] +(63) CometHashAggregate +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(63) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] - -(64) CometFilter -Input [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Condition : isnotnull(ws_bill_customer_sk#36) +(64) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct -(65) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#40] +(65) CometFilter +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) -(66) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [1]: [item_sk#40] -Arguments: [ws_item_sk#35], [item_sk#40], LeftSemi, BuildRight +(66) ReusedExchange [Reuses operator id: 21] +Output [1]: [item_sk#42] -(67) CometProject -Input [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39], [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] +(67) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#42] +Arguments: [ws_item_sk#37], [item_sk#42], LeftSemi, BuildRight -(68) CometExchange -Input [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: hashpartitioning(ws_bill_customer_sk#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(68) CometProject +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] -(69) CometSort -Input [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39], [ws_bill_customer_sk#36 ASC NULLS FIRST] +(69) CometExchange +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(70) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#41, sum#42, isEmpty#43] +(70) CometSort +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] -(71) CometHashAggregate -Input [3]: [c_customer_sk#41, sum#42, isEmpty#43] -Keys [1]: [c_customer_sk#41] -Functions [1]: [sum((cast(ss_quantity#44 as decimal(10,0)) * ss_sales_price#45))] +(71) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#43, sum#44, isEmpty#45] -(72) CometFilter -Input [2]: [c_customer_sk#41, ssales#46] -Condition : (isnotnull(ssales#46) AND (cast(ssales#46 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) +(72) CometHashAggregate +Input [3]: [c_customer_sk#43, sum#44, isEmpty#45] +Keys [1]: [c_customer_sk#43] +Functions [1]: [sum((cast(ss_quantity#46 as decimal(10,0)) * ss_sales_price#47))] -(73) CometProject -Input [2]: [c_customer_sk#41, ssales#46] -Arguments: [c_customer_sk#41], [c_customer_sk#41] +(73) CometFilter +Input [2]: [c_customer_sk#43, ssales#48] +Condition : (isnotnull(ssales#48) AND (cast(ssales#48 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) -(74) CometSort -Input [1]: [c_customer_sk#41] -Arguments: [c_customer_sk#41], [c_customer_sk#41 ASC NULLS FIRST] +(74) CometProject +Input [2]: [c_customer_sk#43, ssales#48] +Arguments: [c_customer_sk#43], [c_customer_sk#43] -(75) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [1]: [c_customer_sk#41] -Arguments: [ws_bill_customer_sk#36], [c_customer_sk#41], LeftSemi +(75) CometSort +Input [1]: [c_customer_sk#43] +Arguments: [c_customer_sk#43], [c_customer_sk#43 ASC NULLS FIRST] -(76) ReusedExchange [Reuses operator id: 51] -Output [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49] +(76) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#43] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#43], LeftSemi -(77) CometBroadcastHashJoin -Left output [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49] -Arguments: [ws_bill_customer_sk#36], [c_customer_sk#47], Inner, BuildRight +(77) ReusedExchange [Reuses operator id: 52] +Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] -(78) CometProject -Input [7]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_customer_sk#47, c_first_name#48, c_last_name#49] -Arguments: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49], [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49] +(78) CometBroadcastHashJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#49], Inner, BuildRight -(79) ReusedExchange [Reuses operator id: 57] -Output [1]: [d_date_sk#50] +(79) CometProject +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] -(80) CometBroadcastHashJoin -Left output [5]: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49] -Right output [1]: [d_date_sk#50] -Arguments: [ws_sold_date_sk#39], [d_date_sk#50], Inner, BuildRight +(80) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#52] -(81) CometProject -Input [6]: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49, d_date_sk#50] -Arguments: [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49], [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49] +(81) CometBroadcastHashJoin +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] +Right output [1]: [d_date_sk#52] +Arguments: [ws_sold_date_sk#41], [d_date_sk#52], Inner, BuildRight -(82) CometHashAggregate -Input [4]: [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49] -Keys [2]: [c_last_name#49, c_first_name#48] -Functions [1]: [partial_sum((cast(ws_quantity#37 as decimal(10,0)) * ws_list_price#38))] +(82) CometProject +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51, d_date_sk#52] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] -(83) CometExchange -Input [4]: [c_last_name#49, c_first_name#48, sum#51, isEmpty#52] -Arguments: hashpartitioning(c_last_name#49, c_first_name#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(83) CometHashAggregate +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] -(84) CometHashAggregate -Input [4]: [c_last_name#49, c_first_name#48, sum#51, isEmpty#52] -Keys [2]: [c_last_name#49, c_first_name#48] -Functions [1]: [sum((cast(ws_quantity#37 as decimal(10,0)) * ws_list_price#38))] +(84) CometExchange +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(85) CometUnion -Child 0 Input [3]: [c_last_name#29, c_first_name#28, sales#53] -Child 1 Input [3]: [c_last_name#49, c_first_name#48, sales#54] +(85) CometHashAggregate +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] -(86) CometTakeOrderedAndProject -Input [3]: [c_last_name#29, c_first_name#28, sales#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#29 ASC NULLS FIRST,c_first_name#28 ASC NULLS FIRST,sales#53 ASC NULLS FIRST], output=[c_last_name#29,c_first_name#28,sales#53]), [c_last_name#29, c_first_name#28, sales#53], 100, [c_last_name#29 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, sales#53 ASC NULLS FIRST], [c_last_name#29, c_first_name#28, sales#53] +(86) CometUnion +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Child 1 Input [3]: [c_last_name#51, c_first_name#50, sales#56] -(87) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#29, c_first_name#28, sales#53] +(87) CometTakeOrderedAndProject +Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#55 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#55]), [c_last_name#31, c_first_name#30, sales#55], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#55 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#55] + +(88) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] ===== Subqueries ===== Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] -* ColumnarToRow (105) -+- CometHashAggregate (104) - +- CometExchange (103) - +- CometHashAggregate (102) - +- CometHashAggregate (101) - +- CometExchange (100) - +- CometHashAggregate (99) - +- CometProject (98) - +- CometBroadcastHashJoin (97) - :- CometProject (92) - : +- CometBroadcastHashJoin (91) - : :- CometFilter (89) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (88) - : +- ReusedExchange (90) - +- CometBroadcastExchange (96) - +- CometProject (95) - +- CometFilter (94) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (93) - - -(88) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Arguments: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] - -(89) CometFilter -Input [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_customer_sk#55) - -(90) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#59] - -(91) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Right output [1]: [c_customer_sk#59] -Arguments: [ss_customer_sk#55], [c_customer_sk#59], Inner, BuildRight - -(92) CometProject -Input [5]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] -Arguments: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59], [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] - -(93) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#60, d_year#61] -Arguments: [d_date_sk#60, d_year#61] - -(94) CometFilter -Input [2]: [d_date_sk#60, d_year#61] -Condition : (d_year#61 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#60)) - -(95) CometProject -Input [2]: [d_date_sk#60, d_year#61] -Arguments: [d_date_sk#60], [d_date_sk#60] - -(96) CometBroadcastExchange -Input [1]: [d_date_sk#60] -Arguments: [d_date_sk#60] - -(97) CometBroadcastHashJoin -Left output [4]: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] -Right output [1]: [d_date_sk#60] -Arguments: [ss_sold_date_sk#58], [d_date_sk#60], Inner, BuildRight - -(98) CometProject -Input [5]: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59, d_date_sk#60] -Arguments: [ss_quantity#56, ss_sales_price#57, c_customer_sk#59], [ss_quantity#56, ss_sales_price#57, c_customer_sk#59] - -(99) CometHashAggregate -Input [3]: [ss_quantity#56, ss_sales_price#57, c_customer_sk#59] -Keys [1]: [c_customer_sk#59] -Functions [1]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_sales_price#57))] - -(100) CometExchange -Input [3]: [c_customer_sk#59, sum#62, isEmpty#63] -Arguments: hashpartitioning(c_customer_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(101) CometHashAggregate -Input [3]: [c_customer_sk#59, sum#62, isEmpty#63] -Keys [1]: [c_customer_sk#59] -Functions [1]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_sales_price#57))] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometNativeScan parquet spark_catalog.default.date_dim (94) + + +(89) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#60)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(90) CometFilter +Input [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Condition : isnotnull(ss_customer_sk#57) + +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#61] + +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Right output [1]: [c_customer_sk#61] +Arguments: [ss_customer_sk#57], [c_customer_sk#61], Inner, BuildRight + +(93) CometProject +Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] + +(94) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#62, d_year#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(95) CometFilter +Input [2]: [d_date_sk#62, d_year#63] +Condition : (d_year#63 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#62)) + +(96) CometProject +Input [2]: [d_date_sk#62, d_year#63] +Arguments: [d_date_sk#62], [d_date_sk#62] + +(97) CometBroadcastExchange +Input [1]: [d_date_sk#62] +Arguments: [d_date_sk#62] + +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#60], [d_date_sk#62], Inner, BuildRight + +(99) CometProject +Input [5]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61, d_date_sk#62] +Arguments: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] + +(100) CometHashAggregate +Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] +Keys [1]: [c_customer_sk#61] +Functions [1]: [partial_sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(101) CometExchange +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Arguments: hashpartitioning(c_customer_sk#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (102) CometHashAggregate -Input [1]: [csales#64] +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Keys [1]: [c_customer_sk#61] +Functions [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(103) CometHashAggregate +Input [1]: [csales#66] Keys: [] -Functions [1]: [partial_max(csales#64)] +Functions [1]: [partial_max(csales#66)] -(103) CometExchange -Input [1]: [max#65] +(104) CometExchange +Input [1]: [max#67] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(104) CometHashAggregate -Input [1]: [max#65] +(105) CometHashAggregate +Input [1]: [max#67] Keys: [] -Functions [1]: [max(csales#64)] +Functions [1]: [max(csales#66)] -(105) ColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#66] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#68] Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:3 Hosting operator id = 72 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/extended.txt new file mode 100644 index 0000000000..bdae9844c6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/extended.txt @@ -0,0 +1,173 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometSort + : : : +- CometProject + : : : +- CometFilter + : : : : +- Subquery + : : : : +- CometColumnarToRow + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometSort + : +- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 165 out of 169 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/simplified.txt index 85aa2ac471..2c389585a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_last_name,c_first_name,sales] CometUnion [c_last_name,c_first_name,sales] - CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] CometExchange [c_last_name,c_first_name] #1 - CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,cs_quantity,cs_list_price] + CometHashAggregate [cs_quantity,cs_list_price] [c_last_name,c_first_name,sum,isEmpty] CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] @@ -16,11 +16,11 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] @@ -28,68 +28,69 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [tpcds_cmax,max,max(csales)] + CometHashAggregate [max] [tpcds_cmax,max(csales)] CometExchange #9 - CometHashAggregate [max,csales] - CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #10 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] ReusedExchange [c_customer_sk] #8 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #7 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 - CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] - CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #13 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - CometSort [c_customer_sk] - CometProject [c_customer_sk] - CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] + CometSort [c_customer_sk,c_first_name,c_last_name] + CometExchange [c_customer_sk] #13 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] CometExchange [c_last_name,c_first_name] #15 - CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,ws_quantity,ws_list_price] + CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] @@ -100,13 +101,13 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] ReusedExchange [c_customer_sk,sum,isEmpty] #7 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/explain.txt index 5d63cafbe5..67a43617c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] @@ -63,9 +71,12 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -92,291 +103,337 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometNativeScan: `spark_catalog`.`default`.`store` +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) + +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] + +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] + +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] + +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometNativeScan parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] (55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight (56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] +(57) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) (59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] + +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] + +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight + +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] + +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] + +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#69] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#69] Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] +Functions [1]: [partial_avg(netpaid#69)] +Aggregate Attributes [2]: [sum#70, count#71] +Results [2]: [sum#72, count#73] + +(75) CometColumnarExchange +Input [2]: [sum#72, count#73] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#72, count#73] -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#72, count#73] Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] +Functions [1]: [avg(netpaid#69)] +Aggregate Attributes [1]: [avg(netpaid#69)#74] +Results [1]: [(0.05 * avg(netpaid#69)#74) AS (0.05 * avg(netpaid))#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/extended.txt new file mode 100644 index 0000000000..6b21db37a2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/extended.txt @@ -0,0 +1,96 @@ +Filter +: +- Subquery +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- Project +: +- BroadcastHashJoin +: :- CometColumnarToRow +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometSortMergeJoin +: : : : : :- CometSort +: : : : : : +- CometExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometSort +: : : : : +- CometExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.item +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.customer +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.customer_address ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/simplified.txt index c1bf41667f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/simplified.txt @@ -3,18 +3,56 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] @@ -25,62 +63,32 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/explain.txt index 829f96c1fb..65c5d9d521 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] @@ -63,9 +71,12 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -92,291 +103,337 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometNativeScan: `spark_catalog`.`default`.`store` +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = chiffon ) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) + +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] + +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] + +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] + +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometNativeScan parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] (55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight (56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] +(57) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) (59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] + +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] + +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight + +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] + +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] + +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#69] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#69] Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] +Functions [1]: [partial_avg(netpaid#69)] +Aggregate Attributes [2]: [sum#70, count#71] +Results [2]: [sum#72, count#73] + +(75) CometColumnarExchange +Input [2]: [sum#72, count#73] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#72, count#73] -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#72, count#73] Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] +Functions [1]: [avg(netpaid#69)] +Aggregate Attributes [1]: [avg(netpaid#69)#74] +Results [1]: [(0.05 * avg(netpaid#69)#74) AS (0.05 * avg(netpaid))#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/extended.txt new file mode 100644 index 0000000000..6b21db37a2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/extended.txt @@ -0,0 +1,96 @@ +Filter +: +- Subquery +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- Project +: +- BroadcastHashJoin +: :- CometColumnarToRow +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometSortMergeJoin +: : : : : :- CometSort +: : : : : : +- CometExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometSort +: : : : : +- CometExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.item +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.customer +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.customer_address ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/simplified.txt index c1bf41667f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/simplified.txt @@ -3,18 +3,56 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] @@ -25,62 +63,32 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/explain.txt index ec5fe33185..d2e939602b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (40) -+- CometTakeOrderedAndProject (39) - +- CometHashAggregate (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) : :- CometProject (27) : : +- CometBroadcastHashJoin (26) : : :- CometProject (24) @@ -19,39 +19,51 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- ReusedExchange (33) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.item (34) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] @@ -70,9 +82,13 @@ Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] @@ -91,9 +107,12 @@ Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] @@ -116,9 +135,12 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] @@ -153,57 +175,80 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -(28) CometNativeScan: `spark_catalog`.`default`.`store` +(28) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (29) CometFilter Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Condition : isnotnull(s_store_sk#23) -(30) CometBroadcastExchange +(30) CometProject Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#26, s_store_name#25] -(31) CometBroadcastHashJoin +(31) CometBroadcastExchange +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] + +(32) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -Right output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight -(32) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] - -(33) ReusedExchange [Reuses operator id: 30] -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] - -(34) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Right output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_item_sk#1], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] - -(36) CometHashAggregate -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(33) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] + +(34) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(35) CometFilter +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) + +(36) CometProject +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] + +(37) CometBroadcastExchange +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] + +(38) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight + +(39) CometProject +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] + +(40) CometHashAggregate +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] -(37) CometExchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#29, sum#30, sum#31] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(41) CometExchange +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(38) CometHashAggregate -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#29, sum#30, sum#31] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(42) CometHashAggregate +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] -(39) CometTakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#28 ASC NULLS FIRST,s_store_id#24 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#28,s_store_id#24,s_store_name#25,store_sales_profit#32,store_returns_loss#33,catalog_sales_profit#34]), [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34], 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] +(43) CometTakeOrderedAndProject +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#26 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_store_id#26,s_store_name#25,store_sales_profit#34,store_returns_loss#35,catalog_sales_profit#36]), [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] -(40) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] +(44) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/simplified.txt index 0074eafcf3..49a567844b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_net_profit,sr_net_loss,cs_net_profit] + CometHashAggregate [ss_net_profit,sr_net_loss,cs_net_profit] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] @@ -20,23 +20,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] ReusedExchange [d_date_sk] #5 CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #6 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/explain.txt index a203045086..17aa7150fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.promotion (21) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Arguments: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -64,9 +72,12 @@ Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#13, d_year#14] @@ -89,70 +100,80 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] +Right output [2]: [i_item_sk#15, i_item_id#17] Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [cs_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(26) CometHashAggregate -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(20) CometProject +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] + +(21) CometNativeScan parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) + +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] + +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] + +(25) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight + +(26) CometProject +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] + +(27) CometHashAggregate +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -(27) CometExchange -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) CometHashAggregate -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] -(29) CometTakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#16 ASC NULLS FIRST], output=[i_item_id#16,agg1#28,agg2#29,agg3#30,agg4#31]), [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31], 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] -(30) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/extended.txt new file mode 100644 index 0000000000..7382341430 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/simplified.txt index 09d0fd2812..55cf7624f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] @@ -14,19 +14,20 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/explain.txt index 05a0abf815..75682e065f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/explain.txt @@ -1,51 +1,60 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometExpand (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (32) ++- CometTakeOrderedAndProject (31) + +- CometHashAggregate (30) + +- CometExchange (29) + +- CometHashAggregate (28) + +- CometExpand (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- CometBroadcastExchange (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.item (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -64,9 +73,12 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#13, d_year#14] @@ -89,70 +101,84 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] +Right output [2]: [s_store_sk#15, s_state#17] Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] +(21) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(25) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] -(26) CometHashAggregate -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(27) CometExpand +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] + +(28) CometHashAggregate +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometExchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(29) CometExchange +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) CometHashAggregate -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(30) CometHashAggregate +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -(29) CometTakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#19 ASC NULLS FIRST,s_state#20 ASC NULLS FIRST], output=[i_item_id#19,s_state#20,g_state#30,agg1#31,agg2#32,agg3#33,agg4#34]), [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34], 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] +(31) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] -(30) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] +(32) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/extended.txt new file mode 100644 index 0000000000..666d08fb88 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/extended.txt @@ -0,0 +1,34 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 31 out of 31 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/simplified.txt index d1c71c8dc4..ff5133d342 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] CometExchange [i_item_id,s_state,spark_grouping_id] #1 - CometHashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] @@ -15,18 +15,20 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/explain.txt index e537028de2..985274408b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/explain.txt @@ -1,79 +1,88 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- * ColumnarToRow (6) -: : : : : +- CometExchange (5) -: : : : : +- CometHashAggregate (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- * ColumnarToRow (16) -: : : : +- CometExchange (15) -: : : : +- CometHashAggregate (14) -: : : : +- CometProject (13) -: : : : +- CometFilter (12) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- * ColumnarToRow (28) -: : : +- CometExchange (27) -: : : +- CometHashAggregate (26) -: : : +- CometProject (25) -: : : +- CometFilter (24) -: : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- * ColumnarToRow (40) -: : +- CometExchange (39) -: : +- CometHashAggregate (38) -: : +- CometProject (37) -: : +- CometFilter (36) -: : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- * ColumnarToRow (52) -: +- CometExchange (51) -: +- CometHashAggregate (50) -: +- CometProject (49) -: +- CometFilter (48) -: +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- * ColumnarToRow (64) - +- CometExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometFilter (60) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (59) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* BroadcastNestedLoopJoin Inner BuildRight (76) +:- * BroadcastNestedLoopJoin Inner BuildRight (63) +: :- * BroadcastNestedLoopJoin Inner BuildRight (50) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (37) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (24) +: : : : :- * CometColumnarToRow (11) +: : : : : +- CometHashAggregate (10) +: : : : : +- CometColumnarExchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * CometColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (23) +: : : : +- * CometColumnarToRow (22) +: : : : +- CometHashAggregate (21) +: : : : +- CometColumnarExchange (20) +: : : : +- * HashAggregate (19) +: : : : +- * HashAggregate (18) +: : : : +- * CometColumnarToRow (17) +: : : : +- CometExchange (16) +: : : : +- CometHashAggregate (15) +: : : : +- CometProject (14) +: : : : +- CometFilter (13) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (12) +: : : +- BroadcastExchange (36) +: : : +- * CometColumnarToRow (35) +: : : +- CometHashAggregate (34) +: : : +- CometColumnarExchange (33) +: : : +- * HashAggregate (32) +: : : +- * HashAggregate (31) +: : : +- * CometColumnarToRow (30) +: : : +- CometExchange (29) +: : : +- CometHashAggregate (28) +: : : +- CometProject (27) +: : : +- CometFilter (26) +: : : +- CometNativeScan parquet spark_catalog.default.store_sales (25) +: : +- BroadcastExchange (49) +: : +- * CometColumnarToRow (48) +: : +- CometHashAggregate (47) +: : +- CometColumnarExchange (46) +: : +- * HashAggregate (45) +: : +- * HashAggregate (44) +: : +- * CometColumnarToRow (43) +: : +- CometExchange (42) +: : +- CometHashAggregate (41) +: : +- CometProject (40) +: : +- CometFilter (39) +: : +- CometNativeScan parquet spark_catalog.default.store_sales (38) +: +- BroadcastExchange (62) +: +- * CometColumnarToRow (61) +: +- CometHashAggregate (60) +: +- CometColumnarExchange (59) +: +- * HashAggregate (58) +: +- * HashAggregate (57) +: +- * CometColumnarToRow (56) +: +- CometExchange (55) +: +- CometHashAggregate (54) +: +- CometProject (53) +: +- CometFilter (52) +: +- CometNativeScan parquet spark_catalog.default.store_sales (51) ++- BroadcastExchange (75) + +- * CometColumnarToRow (74) + +- CometHashAggregate (73) + +- CometColumnarExchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- * CometColumnarToRow (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometProject (66) + +- CometFilter (65) + +- CometNativeScan parquet spark_catalog.default.store_sales (64) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Arguments: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))] +ReadSchema: struct (2) CometFilter Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] @@ -92,7 +101,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_li Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) ColumnarToRow [codegen id : 1] +(6) CometColumnarToRow [codegen id : 1] Input [4]: [ss_list_price#3, sum#6, count#7, count#8] (7) HashAggregate [codegen id : 1] @@ -109,299 +118,320 @@ Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_p Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] Results [4]: [sum#6, count#7, count#8, count#12] -(9) Exchange +(9) CometColumnarExchange Input [4]: [sum#6, count#7, count#8, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(10) HashAggregate [codegen id : 12] +(10) CometHashAggregate Input [4]: [sum#6, count#7, count#8, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#9 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#10 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(11) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(11) CometColumnarToRow [codegen id : 12] +Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] + +(12) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] +ReadSchema: struct -(12) CometFilter +(13) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(13) CometProject +(14) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Arguments: [ss_list_price#18], [ss_list_price#18] -(14) CometHashAggregate +(15) CometHashAggregate Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -(15) CometExchange +(16) CometExchange Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(16) ColumnarToRow [codegen id : 2] +(17) CometColumnarToRow [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -(17) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] Results [4]: [ss_list_price#18, sum#21, count#22, count#23] -(18) HashAggregate [codegen id : 2] +(19) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] Results [4]: [sum#21, count#22, count#23, count#27] -(19) Exchange +(20) CometColumnarExchange Input [4]: [sum#21, count#22, count#23, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(20) HashAggregate [codegen id : 3] +(21) CometHashAggregate Input [4]: [sum#21, count#22, count#23, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#25 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(21) BroadcastExchange +(22) CometColumnarToRow [codegen id : 3] +Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] + +(23) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(22) BroadcastNestedLoopJoin [codegen id : 12] +(24) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(23) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(25) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Arguments: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] +ReadSchema: struct -(24) CometFilter +(26) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(25) CometProject +(27) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Arguments: [ss_list_price#33], [ss_list_price#33] -(26) CometHashAggregate +(28) CometHashAggregate Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -(27) CometExchange +(29) CometExchange Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(28) ColumnarToRow [codegen id : 4] +(30) CometColumnarToRow [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -(29) HashAggregate [codegen id : 4] +(31) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] Results [4]: [ss_list_price#33, sum#36, count#37, count#38] -(30) HashAggregate [codegen id : 4] +(32) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] Results [4]: [sum#36, count#37, count#38, count#42] -(31) Exchange +(33) CometColumnarExchange Input [4]: [sum#36, count#37, count#38, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(32) HashAggregate [codegen id : 5] +(34) CometHashAggregate Input [4]: [sum#36, count#37, count#38, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#40 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(33) BroadcastExchange +(35) CometColumnarToRow [codegen id : 5] +Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] + +(36) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(34) BroadcastNestedLoopJoin [codegen id : 12] +(37) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(35) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(38) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Arguments: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] +ReadSchema: struct -(36) CometFilter +(39) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(37) CometProject +(40) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Arguments: [ss_list_price#48], [ss_list_price#48] -(38) CometHashAggregate +(41) CometHashAggregate Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -(39) CometExchange +(42) CometExchange Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(40) ColumnarToRow [codegen id : 6] +(43) CometColumnarToRow [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -(41) HashAggregate [codegen id : 6] +(44) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] Results [4]: [ss_list_price#48, sum#51, count#52, count#53] -(42) HashAggregate [codegen id : 6] +(45) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] Results [4]: [sum#51, count#52, count#53, count#57] -(43) Exchange +(46) CometColumnarExchange Input [4]: [sum#51, count#52, count#53, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(44) HashAggregate [codegen id : 7] +(47) CometHashAggregate Input [4]: [sum#51, count#52, count#53, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#55 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(45) BroadcastExchange +(48) CometColumnarToRow [codegen id : 7] +Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] + +(49) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(46) BroadcastNestedLoopJoin [codegen id : 12] +(50) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(51) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Arguments: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] +ReadSchema: struct -(48) CometFilter +(52) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(49) CometProject +(53) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Arguments: [ss_list_price#63], [ss_list_price#63] -(50) CometHashAggregate +(54) CometHashAggregate Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -(51) CometExchange +(55) CometExchange Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(52) ColumnarToRow [codegen id : 8] +(56) CometColumnarToRow [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -(53) HashAggregate [codegen id : 8] +(57) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] Results [4]: [ss_list_price#63, sum#66, count#67, count#68] -(54) HashAggregate [codegen id : 8] +(58) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] Results [4]: [sum#66, count#67, count#68, count#72] -(55) Exchange +(59) CometColumnarExchange Input [4]: [sum#66, count#67, count#68, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] -(56) HashAggregate [codegen id : 9] +(60) CometHashAggregate Input [4]: [sum#66, count#67, count#68, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#70 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(57) BroadcastExchange +(61) CometColumnarToRow [codegen id : 9] +Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] + +(62) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(58) BroadcastNestedLoopJoin [codegen id : 12] +(63) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(59) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(64) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Arguments: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] +ReadSchema: struct -(60) CometFilter +(65) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(61) CometProject +(66) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Arguments: [ss_list_price#78], [ss_list_price#78] -(62) CometHashAggregate +(67) CometHashAggregate Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -(63) CometExchange +(68) CometExchange Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(64) ColumnarToRow [codegen id : 10] +(69) CometColumnarToRow [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -(65) HashAggregate [codegen id : 10] +(70) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] Results [4]: [ss_list_price#78, sum#81, count#82, count#83] -(66) HashAggregate [codegen id : 10] +(71) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] Results [4]: [sum#81, count#82, count#83, count#87] -(67) Exchange +(72) CometColumnarExchange Input [4]: [sum#81, count#82, count#83, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] -(68) HashAggregate [codegen id : 11] +(73) CometHashAggregate Input [4]: [sum#81, count#82, count#83, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#85 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(69) BroadcastExchange +(74) CometColumnarToRow [codegen id : 11] +Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] + +(75) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(70) BroadcastNestedLoopJoin [codegen id : 12] +(76) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/extended.txt new file mode 100644 index 0000000000..3d8c2f298e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +BroadcastNestedLoopJoin +:- BroadcastNestedLoopJoin +: :- BroadcastNestedLoopJoin +: : :- BroadcastNestedLoopJoin +: : : :- BroadcastNestedLoopJoin +: : : : :- CometColumnarToRow +: : : : : +- CometHashAggregate +: : : : : +- CometColumnarExchange +: : : : : +- HashAggregate +: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : : +- CometColumnarToRow +: : : : : +- CometExchange +: : : : : +- CometHashAggregate +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : +- BroadcastExchange +: : : : +- CometColumnarToRow +: : : : +- CometHashAggregate +: : : : +- CometColumnarExchange +: : : : +- HashAggregate +: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : +- CometColumnarToRow +: : : : +- CometExchange +: : : : +- CometHashAggregate +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : +- BroadcastExchange +: : : +- CometColumnarToRow +: : : +- CometHashAggregate +: : : +- CometColumnarExchange +: : : +- HashAggregate +: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : +- CometColumnarToRow +: : : +- CometExchange +: : : +- CometHashAggregate +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : +- BroadcastExchange +: : +- CometColumnarToRow +: : +- CometHashAggregate +: : +- CometColumnarExchange +: : +- HashAggregate +: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : +- CometColumnarToRow +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometHashAggregate +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: +- CometColumnarToRow +: +- CometExchange +: +- CometHashAggregate +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.store_sales ++- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_sales + +Comet accelerated 42 out of 64 eligible operators (65%). Final plan contains 12 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/simplified.txt index 03812cf5b6..a8540a4abb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/simplified.txt @@ -4,96 +4,102 @@ WholeStageCodegen (12) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #2 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B1_LP,B1_CNT,B1_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (3) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] + CometColumnarToRow InputAdapter - Exchange #4 - WholeStageCodegen (2) + CometHashAggregate [sum,count,count,count] [B2_LP,B2_CNT,B2_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #4 + WholeStageCodegen (2) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #5 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B3_LP,B3_CNT,B3_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #7 + WholeStageCodegen (4) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #5 + CometExchange [ss_list_price] #8 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] - InputAdapter - Exchange #7 - WholeStageCodegen (4) + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B4_LP,B4_CNT,B4_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #10 + WholeStageCodegen (6) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #8 + CometExchange [ss_list_price] #11 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] - InputAdapter - Exchange #10 - WholeStageCodegen (6) + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (9) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B5_LP,B5_CNT,B5_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #13 + WholeStageCodegen (8) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #11 + CometExchange [ss_list_price] #14 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] - InputAdapter - Exchange #13 - WholeStageCodegen (8) + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B6_LP,B6_CNT,B6_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #16 + WholeStageCodegen (10) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #14 + CometExchange [ss_list_price] #17 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] - InputAdapter - Exchange #16 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #17 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/explain.txt index bfe4cc564b..4b72e29f59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (43) -+- CometTakeOrderedAndProject (42) - +- CometHashAggregate (41) - +- CometExchange (40) - +- CometHashAggregate (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) : :- CometProject (30) : : +- CometBroadcastHashJoin (29) : : :- CometProject (24) @@ -19,42 +19,54 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- CometBroadcastExchange (28) : : +- CometProject (27) : : +- CometFilter (26) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (31) - +- ReusedExchange (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometNativeScan parquet spark_catalog.default.store (31) + +- CometBroadcastExchange (40) + +- CometProject (39) + +- CometFilter (38) + +- CometNativeScan parquet spark_catalog.default.item (37) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] @@ -73,9 +85,13 @@ Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] @@ -94,9 +110,12 @@ Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] @@ -119,9 +138,12 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] @@ -144,9 +166,12 @@ Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (26) CometFilter Input [2]: [d_date_sk#22, d_year#23] @@ -169,57 +194,80 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(31) CometNativeScan: `spark_catalog`.`default`.`store` +(31) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (32) CometFilter Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Condition : isnotnull(s_store_sk#24) -(33) CometBroadcastExchange +(33) CometProject Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#27, s_store_name#26] + +(34) CometBroadcastExchange +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] -(34) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight -(35) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] - -(36) ReusedExchange [Reuses operator id: 33] -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] - -(37) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Right output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight - -(38) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] - -(39) CometHashAggregate -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(36) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] + +(37) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(38) CometFilter +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) + +(39) CometProject +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#31, i_item_desc#30] + +(40) CometBroadcastExchange +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] + +(41) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight + +(42) CometProject +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] + +(43) CometHashAggregate +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] -(40) CometExchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(44) CometExchange +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(41) CometHashAggregate -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#30, sum#31, sum#32] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(45) CometHashAggregate +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] -(42) CometTakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#28 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#25 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#28,i_item_desc#29,s_store_id#25,s_store_name#26,store_sales_quantity#33,store_returns_quantity#34,catalog_sales_quantity#35]), [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35], 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] +(46) CometTakeOrderedAndProject +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] -(43) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] +(47) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/simplified.txt index 33adc3854b..53a498d595 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_quantity,sr_return_quantity,cs_quantity] + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] @@ -20,26 +20,30 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/explain.txt index 527a1e97da..fd084402f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,19 +10,22 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] @@ -63,39 +73,39 @@ Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isno (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] (15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [4]: [d_year#2, brand_id#12, brand#13, sum_agg#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,sum_agg#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[d_year#2,brand_id#12,brand#13,sum_agg#14]), [d_year#2, brand_id#12, brand#13, sum_agg#14], 100, [d_year#2 ASC NULLS FIRST, sum_agg#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [d_year#2, brand_id#12, brand#13, sum_agg#14] +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,sum_agg#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,sum_agg#15]), [d_year#2, brand_id#13, brand#14, sum_agg#15], 100, 0, [d_year#2 ASC NULLS FIRST, sum_agg#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, sum_agg#15] -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, brand_id#12, brand#13, sum_agg#14] +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/simplified.txt index 6828a328ea..c19e6caf52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,sum_agg] - CometHashAggregate [d_year,brand_id,brand,sum_agg,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [d_year,brand_id,brand,sum_agg,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [d_year,i_brand,i_brand_id] #1 - CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] CometProject [d_year,ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/explain.txt index c5873bebbd..a2aa9b30a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/explain.txt @@ -1,65 +1,74 @@ == Physical Plan == -* ColumnarToRow (48) -+- CometTakeOrderedAndProject (47) - +- CometProject (46) - +- CometBroadcastHashJoin (45) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometFilter (17) - : : : +- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- CometBroadcastExchange (33) - : : +- CometFilter (32) - : : +- CometHashAggregate (31) - : : +- CometExchange (30) - : : +- CometHashAggregate (29) - : : +- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (18) - : : : +- ReusedExchange (20) - : : +- ReusedExchange (23) - : +- CometBroadcastExchange (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (36) - +- CometBroadcastExchange (44) - +- CometProject (43) - +- CometFilter (42) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (41) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_returns` + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometNativeScan parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) + + +(1) CometNativeScan parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#4)] +PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_year#6] @@ -82,172 +91,193 @@ Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#7, ca_state#8] +Right output [2]: [ca_address_sk#7, ca_state#9] Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] +(14) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] -(14) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -(15) CometExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#9] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#9] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -(17) CometFilter -Input [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) - -(18) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Arguments: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(19) CometFilter -Input [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Condition : isnotnull(wr_returning_addr_sk#14) +(19) CometNativeScan parquet spark_catalog.default.web_returns +Output [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#17)] +PushedFilters: [IsNotNull(wr_returning_addr_sk)] +ReadSchema: struct -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +(20) CometFilter +Input [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] +Condition : isnotnull(wr_returning_addr_sk#15) -(21) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [wr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] -(22) CometProject -Input [5]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16, d_date_sk#17] -Arguments: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15], [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15] +(22) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [wr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight -(23) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#18, ca_state#19] +(23) CometProject +Input [5]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17, d_date_sk#18] +Arguments: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16], [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16] -(24) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15] -Right output [2]: [ca_address_sk#18, ca_state#19] -Arguments: [wr_returning_addr_sk#14], [ca_address_sk#18], Inner, BuildRight +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#19, ca_state#9] -(25) CometProject -Input [5]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, ca_address_sk#18, ca_state#19] -Arguments: [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19], [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19] +(25) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16] +Right output [2]: [ca_address_sk#19, ca_state#9] +Arguments: [wr_returning_addr_sk#15], [ca_address_sk#19], Inner, BuildRight -(26) CometHashAggregate -Input [3]: [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19] -Keys [2]: [wr_returning_customer_sk#13, ca_state#19] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#15))] +(26) CometProject +Input [5]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, ca_address_sk#19, ca_state#9] +Arguments: [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9], [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9] -(27) CometExchange -Input [3]: [wr_returning_customer_sk#13, ca_state#19, sum#20] -Arguments: hashpartitioning(wr_returning_customer_sk#13, ca_state#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometHashAggregate +Input [3]: [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9] +Keys [2]: [wr_returning_customer_sk#14, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#16))] -(28) CometHashAggregate -Input [3]: [wr_returning_customer_sk#13, ca_state#19, sum#20] -Keys [2]: [wr_returning_customer_sk#13, ca_state#19] -Functions [1]: [sum(UnscaledValue(wr_return_amt#15))] +(28) CometExchange +Input [3]: [wr_returning_customer_sk#14, ca_state#9, sum#20] +Arguments: hashpartitioning(wr_returning_customer_sk#14, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate +Input [3]: [wr_returning_customer_sk#14, ca_state#9, sum#20] +Keys [2]: [wr_returning_customer_sk#14, ca_state#9] +Functions [1]: [sum(UnscaledValue(wr_return_amt#16))] + +(30) CometHashAggregate Input [2]: [ctr_state#21, ctr_total_return#22] Keys [1]: [ctr_state#21] Functions [1]: [partial_avg(ctr_total_return#22)] -(30) CometExchange +(31) CometExchange Input [3]: [ctr_state#21, sum#23, count#24] Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(31) CometHashAggregate +(32) CometHashAggregate Input [3]: [ctr_state#21, sum#23, count#24] Keys [1]: [ctr_state#21] Functions [1]: [avg(ctr_total_return#22)] -(32) CometFilter +(33) CometFilter Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) -(33) CometBroadcastExchange +(34) CometBroadcastExchange Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -(34) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_state#11], [ctr_state#21], Inner, (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight +Arguments: [ctr_state#12], [ctr_state#21], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight -(35) CometProject -Input [5]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_customer_sk#10, ctr_total_return#12], [ctr_customer_sk#10, ctr_total_return#12] +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] -(36) CometNativeScan: `spark_catalog`.`default`.`customer` +(37) CometNativeScan parquet spark_catalog.default.customer Output [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct -(37) CometFilter +(38) CometFilter Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) -(38) CometBroadcastExchange +(39) CometProject Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] - -(39) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#10, ctr_total_return#12] -Right output [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [ctr_customer_sk#10], [c_customer_sk#26], Inner, BuildRight - -(40) CometProject -Input [16]: [ctr_customer_sk#10, ctr_total_return#12, c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39], [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] - -(41) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#40, ca_state#41] -Arguments: [ca_address_sk#40, ca_state#41] - -(42) CometFilter -Input [2]: [ca_address_sk#40, ca_state#41] -Condition : ((isnotnull(ca_state#41) AND (ca_state#41 = GA)) AND isnotnull(ca_address_sk#40)) - -(43) CometProject -Input [2]: [ca_address_sk#40, ca_state#41] -Arguments: [ca_address_sk#40], [ca_address_sk#40] - -(44) CometBroadcastExchange -Input [1]: [ca_address_sk#40] -Arguments: [ca_address_sk#40] - -(45) CometBroadcastHashJoin -Left output [14]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Right output [1]: [ca_address_sk#40] -Arguments: [c_current_addr_sk#28], [ca_address_sk#40], Inner, BuildRight - -(46) CometProject -Input [15]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ca_address_sk#40] -Arguments: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] - -(47) CometTakeOrderedAndProject -Input [13]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#27 ASC NULLS FIRST,c_salutation#29 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,c_last_name#31 ASC NULLS FIRST,c_preferred_cust_flag#32 ASC NULLS FIRST,c_birth_day#33 ASC NULLS FIRST,c_birth_month#34 ASC NULLS FIRST,c_birth_year#35 ASC NULLS FIRST,c_birth_country#36 ASC NULLS FIRST,c_login#37 ASC NULLS FIRST,c_email_address#38 ASC NULLS FIRST,c_last_review_date#39 ASC NULLS FIRST,ctr_total_return#12 ASC NULLS FIRST], output=[c_customer_id#27,c_salutation#29,c_first_name#30,c_last_name#31,c_preferred_cust_flag#32,c_birth_day#33,c_birth_month#34,c_birth_year#35,c_birth_country#36,c_login#37,c_email_address#38,c_last_review_date#39,ctr_total_return#12]), [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12], 100, [c_customer_id#27 ASC NULLS FIRST, c_salutation#29 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, c_last_name#31 ASC NULLS FIRST, c_preferred_cust_flag#32 ASC NULLS FIRST, c_birth_day#33 ASC NULLS FIRST, c_birth_month#34 ASC NULLS FIRST, c_birth_year#35 ASC NULLS FIRST, c_birth_country#36 ASC NULLS FIRST, c_login#37 ASC NULLS FIRST, c_email_address#38 ASC NULLS FIRST, c_last_review_date#39 ASC NULLS FIRST, ctr_total_return#12 ASC NULLS FIRST], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] - -(48) ColumnarToRow [codegen id : 1] -Input [13]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] +Arguments: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#27, 16, true, false, true) AS c_customer_id#40, c_current_addr_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#29, 10, true, false, true) AS c_salutation#41, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#43, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#32, 1, true, false, true) AS c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#37, 13, true, false, true) AS c_login#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#38, 50, true, false, true) AS c_email_address#46, c_last_review_date#39] + +(40) CometBroadcastExchange +Input [14]: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] + +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [14]: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [ctr_customer_sk#11], [c_customer_sk#26], Inner, BuildRight + +(42) CometProject +Input [16]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39], [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] + +(43) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#47, ca_state#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(44) CometFilter +Input [2]: [ca_address_sk#47, ca_state#48] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#48, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#47)) + +(45) CometProject +Input [2]: [ca_address_sk#47, ca_state#48] +Arguments: [ca_address_sk#47], [ca_address_sk#47] + +(46) CometBroadcastExchange +Input [1]: [ca_address_sk#47] +Arguments: [ca_address_sk#47] + +(47) CometBroadcastHashJoin +Left output [14]: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Right output [1]: [ca_address_sk#47] +Arguments: [c_current_addr_sk#28], [ca_address_sk#47], Inner, BuildRight + +(48) CometProject +Input [15]: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ca_address_sk#47] +Arguments: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13], [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] + +(49) CometTakeOrderedAndProject +Input [13]: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#40 ASC NULLS FIRST,c_salutation#41 ASC NULLS FIRST,c_first_name#42 ASC NULLS FIRST,c_last_name#43 ASC NULLS FIRST,c_preferred_cust_flag#44 ASC NULLS FIRST,c_birth_day#33 ASC NULLS FIRST,c_birth_month#34 ASC NULLS FIRST,c_birth_year#35 ASC NULLS FIRST,c_birth_country#36 ASC NULLS FIRST,c_login#45 ASC NULLS FIRST,c_email_address#46 ASC NULLS FIRST,c_last_review_date#39 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#40,c_salutation#41,c_first_name#42,c_last_name#43,c_preferred_cust_flag#44,c_birth_day#33,c_birth_month#34,c_birth_year#35,c_birth_country#36,c_login#45,c_email_address#46,c_last_review_date#39,ctr_total_return#13]), [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13], 100, 0, [c_customer_id#40 ASC NULLS FIRST, c_salutation#41 ASC NULLS FIRST, c_first_name#42 ASC NULLS FIRST, c_last_name#43 ASC NULLS FIRST, c_preferred_cust_flag#44 ASC NULLS FIRST, c_birth_day#33 ASC NULLS FIRST, c_birth_month#34 ASC NULLS FIRST, c_birth_year#35 ASC NULLS FIRST, c_birth_country#36 ASC NULLS FIRST, c_login#45 ASC NULLS FIRST, c_email_address#46 ASC NULLS FIRST, c_last_review_date#39 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] + +(50) CometColumnarToRow [codegen id : 1] +Input [13]: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/extended.txt new file mode 100644 index 0000000000..d2bf365243 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/extended.txt @@ -0,0 +1,58 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 55 out of 55 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/simplified.txt index d36117ef8e..e57f0ef9b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] @@ -9,42 +9,44 @@ WholeStageCodegen (1) CometProject [ctr_customer_sk,ctr_total_return] CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum(UnscaledValue(wr_return_amt))] CometExchange [wr_returning_customer_sk,ca_state] #1 - CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return)] CometExchange [ctr_state] #5 - CometHashAggregate [ctr_state,sum,count,ctr_total_return] - CometHashAggregate [ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [wr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(wr_return_amt))] CometExchange [wr_returning_customer_sk,ca_state] #6 - CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] ReusedExchange [d_date_sk] #2 ReusedExchange [ca_address_sk,ca_state] #3 CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #7 - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometBroadcastExchange [ca_address_sk] #8 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/explain.txt index efa61c768a..708c0ce9e9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/explain.txt @@ -1,12 +1,12 @@ == Physical Plan == -* ColumnarToRow (59) -+- CometSort (58) - +- CometColumnarExchange (57) - +- CometProject (56) - +- CometBroadcastHashJoin (55) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (50) +* CometColumnarToRow (90) ++- CometSort (89) + +- CometExchange (88) + +- CometProject (87) + +- CometBroadcastHashJoin (86) + :- CometProject (73) + : +- CometBroadcastHashJoin (72) + : :- CometBroadcastHashJoin (59) : : :- CometProject (46) : : : +- CometBroadcastHashJoin (45) : : : :- CometBroadcastHashJoin (30) @@ -18,13 +18,13 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : : : +- CometBroadcastExchange (29) : : : : +- CometHashAggregate (28) : : : : +- CometExchange (27) @@ -34,10 +34,10 @@ : : : : :- CometProject (22) : : : : : +- CometBroadcastHashJoin (21) : : : : : :- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (16) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (16) : : : : : +- CometBroadcastExchange (20) : : : : : +- CometFilter (19) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (18) : : : : +- ReusedExchange (23) : : : +- CometBroadcastExchange (44) : : : +- CometHashAggregate (43) @@ -48,29 +48,67 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (32) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (31) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (31) : : : : +- CometBroadcastExchange (35) : : : : +- CometFilter (34) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (33) : : : +- ReusedExchange (38) - : : +- CometBroadcastExchange (49) - : : +- CometHashAggregate (48) - : : +- ReusedExchange (47) - : +- ReusedExchange (51) - +- ReusedExchange (54) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometBroadcastExchange (58) + : : +- CometHashAggregate (57) + : : +- CometExchange (56) + : : +- CometHashAggregate (55) + : : +- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometFilter (48) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) + : : : +- ReusedExchange (49) + : : +- ReusedExchange (52) + : +- CometBroadcastExchange (71) + : +- CometHashAggregate (70) + : +- CometExchange (69) + : +- CometHashAggregate (68) + : +- CometProject (67) + : +- CometBroadcastHashJoin (66) + : :- CometProject (64) + : : +- CometBroadcastHashJoin (63) + : : :- CometFilter (61) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (60) + : : +- ReusedExchange (62) + : +- ReusedExchange (65) + +- CometBroadcastExchange (85) + +- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometProject (81) + +- CometBroadcastHashJoin (80) + :- CometProject (78) + : +- CometBroadcastHashJoin (77) + : :- CometFilter (75) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (74) + : +- ReusedExchange (76) + +- ReusedExchange (79) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] @@ -89,9 +127,12 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ca_address_sk#7, ca_county#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] +ReadSchema: struct (9) CometFilter Input [2]: [ca_address_sk#7, ca_county#8] @@ -124,17 +165,24 @@ Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(16) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(16) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] -Arguments: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct (17) CometFilter Input [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] Condition : isnotnull(ss_addr_sk#10) -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(18) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13, d_year#14, d_qoy#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (19) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] @@ -188,17 +236,24 @@ Left output [3]: [ca_county#8, d_year#5, store_sales#20] Right output [2]: [ca_county#17, store_sales#19] Arguments: [ca_county#8], [ca_county#17], Inner, BuildRight -(31) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(31) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct (32) CometFilter Input [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Condition : isnotnull(ss_addr_sk#21) -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(33) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#24, d_year#25, d_qoy#26] -Arguments: [d_date_sk#24, d_year#25, d_qoy#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (34) CometFilter Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] @@ -256,55 +311,199 @@ Arguments: [ca_county#17], [ca_county#28], Inner, BuildRight Input [7]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19, ca_county#28, store_sales#30] Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] -(47) ReusedExchange [Reuses operator id: 14] -Output [4]: [ca_county#31, d_qoy#32, d_year#33, sum#34] +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct -(48) CometHashAggregate -Input [4]: [ca_county#31, d_qoy#32, d_year#33, sum#34] -Keys [3]: [ca_county#31, d_qoy#32, d_year#33] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#35))] +(48) CometFilter +Input [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : isnotnull(ws_bill_addr_sk#31) -(49) CometBroadcastExchange -Input [2]: [ca_county#31, web_sales#36] -Arguments: [ca_county#31, web_sales#36] +(49) ReusedExchange [Reuses operator id: 5] +Output [3]: [d_date_sk#34, d_year#35, d_qoy#36] (50) CometBroadcastHashJoin -Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] -Right output [2]: [ca_county#31, web_sales#36] -Arguments: [ca_county#8], [ca_county#31], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Right output [3]: [d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight -(51) ReusedExchange [Reuses operator id: 29] -Output [2]: [ca_county#37, web_sales#38] +(51) CometProject +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36], [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] -(52) CometBroadcastHashJoin -Left output [7]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36] -Right output [2]: [ca_county#37, web_sales#38] -Arguments: [ca_county#31], [ca_county#37], Inner, (CASE WHEN (web_sales#36 > 0.00) THEN (web_sales#38 / web_sales#36) END > CASE WHEN (store_sales#20 > 0.00) THEN (store_sales#19 / store_sales#20) END), BuildRight +(52) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#37, ca_county#38] -(53) CometProject -Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, ca_county#37, web_sales#38] -Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38] +(53) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] +Right output [2]: [ca_address_sk#37, ca_county#38] +Arguments: [ws_bill_addr_sk#31], [ca_address_sk#37], Inner, BuildRight -(54) ReusedExchange [Reuses operator id: 44] -Output [2]: [ca_county#39, web_sales#40] +(54) CometProject +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_address_sk#37, ca_county#38] +Arguments: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38], [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] -(55) CometBroadcastHashJoin -Left output [8]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38] -Right output [2]: [ca_county#39, web_sales#40] -Arguments: [ca_county#31], [ca_county#39], Inner, (CASE WHEN (web_sales#38 > 0.00) THEN (web_sales#40 / web_sales#38) END > CASE WHEN (store_sales#19 > 0.00) THEN (store_sales#30 / store_sales#19) END), BuildRight +(55) CometHashAggregate +Input [4]: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -(56) CometProject -Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38, ca_county#39, web_sales#40] -Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44], [ca_county#8, d_year#5, (web_sales#38 / web_sales#36) AS web_q1_q2_increase#41, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#42, (web_sales#40 / web_sales#38) AS web_q2_q3_increase#43, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#44] +(56) CometExchange +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Arguments: hashpartitioning(ca_county#38, d_qoy#36, d_year#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(57) CometColumnarExchange -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +(57) CometHashAggregate +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -(58) CometSort -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44] -Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44], [ca_county#8 ASC NULLS FIRST] +(58) CometBroadcastExchange +Input [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#38, web_sales#40] -(59) ColumnarToRow [codegen id : 1] -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44] +(59) CometBroadcastHashJoin +Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] +Right output [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#8], [ca_county#38], Inner, BuildRight + +(60) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(61) CometFilter +Input [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_addr_sk#41) + +(62) ReusedExchange [Reuses operator id: 20] +Output [3]: [d_date_sk#44, d_year#45, d_qoy#46] + +(63) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Right output [3]: [d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(64) CometProject +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46], [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] + +(65) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#47, ca_county#48] + +(66) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] +Right output [2]: [ca_address_sk#47, ca_county#48] +Arguments: [ws_bill_addr_sk#41], [ca_address_sk#47], Inner, BuildRight + +(67) CometProject +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_address_sk#47, ca_county#48] +Arguments: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48], [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] + +(68) CometHashAggregate +Input [4]: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#42))] + +(69) CometExchange +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Arguments: hashpartitioning(ca_county#48, d_qoy#46, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(70) CometHashAggregate +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#42))] + +(71) CometBroadcastExchange +Input [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#48, web_sales#50] + +(72) CometBroadcastHashJoin +Left output [7]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40] +Right output [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#38], [ca_county#48], Inner, (CASE WHEN (web_sales#40 > 0.00) THEN (web_sales#50 / web_sales#40) END > CASE WHEN (store_sales#20 > 0.00) THEN (store_sales#19 / store_sales#20) END), BuildRight + +(73) CometProject +Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, ca_county#48, web_sales#50] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] + +(74) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#53)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(75) CometFilter +Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_addr_sk#51) + +(76) ReusedExchange [Reuses operator id: 35] +Output [3]: [d_date_sk#54, d_year#55, d_qoy#56] + +(77) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Right output [3]: [d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_sold_date_sk#53], [d_date_sk#54], Inner, BuildRight + +(78) CometProject +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53, d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56], [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] + +(79) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#57, ca_county#58] + +(80) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] +Right output [2]: [ca_address_sk#57, ca_county#58] +Arguments: [ws_bill_addr_sk#51], [ca_address_sk#57], Inner, BuildRight + +(81) CometProject +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_address_sk#57, ca_county#58] +Arguments: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58], [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] + +(82) CometHashAggregate +Input [4]: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#52))] + +(83) CometExchange +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Arguments: hashpartitioning(ca_county#58, d_qoy#56, d_year#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(84) CometHashAggregate +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#52))] + +(85) CometBroadcastExchange +Input [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#58, web_sales#60] + +(86) CometBroadcastHashJoin +Left output [8]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] +Right output [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#38], [ca_county#58], Inner, (CASE WHEN (web_sales#50 > 0.00) THEN (web_sales#60 / web_sales#50) END > CASE WHEN (store_sales#19 > 0.00) THEN (store_sales#30 / store_sales#19) END), BuildRight + +(87) CometProject +Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50, ca_county#58, web_sales#60] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8, d_year#5, (web_sales#50 / web_sales#40) AS web_q1_q2_increase#61, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#62, (web_sales#60 / web_sales#50) AS web_q2_q3_increase#63, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#64] + +(88) CometExchange +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(89) CometSort +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8 ASC NULLS FIRST] + +(90) CometColumnarToRow [codegen id : 1] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/extended.txt new file mode 100644 index 0000000000..f22c7aa1ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/extended.txt @@ -0,0 +1,108 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- CometBroadcastExchange + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 105 out of 105 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/simplified.txt index 161acbfd53..ef8b87a3b7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] - CometColumnarExchange [ca_county] #1 + CometExchange [ca_county] #1 CometProject [web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales,ca_county,web_sales] CometProject [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] @@ -11,51 +11,82 @@ WholeStageCodegen (1) CometProject [ca_county,d_year,store_sales,store_sales,store_sales] CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales,ca_county,store_sales] CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales] - CometHashAggregate [ca_county,d_year,store_sales,d_qoy,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [d_qoy,sum] [ca_county,d_year,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #2 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [ca_address_sk,ca_county] #4 CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #5 - CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #6 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #4 CometBroadcastExchange [ca_county,store_sales] #8 - CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #9 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #4 CometBroadcastExchange [ca_county,web_sales] #11 - CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [ca_county,d_qoy,d_year,sum] #2 - ReusedExchange [ca_county,web_sales] #5 - ReusedExchange [ca_county,web_sales] #8 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #12 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #13 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #14 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #15 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #16 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/explain.txt index b08d289beb..317132ca06 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (30) +* CometColumnarToRow (30) +- CometHashAggregate (29) +- CometExchange (28) +- CometHashAggregate (27) @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) + : : +- CometNativeScan parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,25 +23,32 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] @@ -64,17 +71,24 @@ Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Arguments: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Condition : isnotnull(cs_item_sk#6) -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] +ReadSchema: struct (12) CometFilter Input [2]: [d_date_sk#9, d_date#10] @@ -154,6 +168,6 @@ Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -(30) ColumnarToRow [codegen id : 1] +(30) CometColumnarToRow [codegen id : 1] Input [1]: [excess discount amount#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/extended.txt new file mode 100644 index 0000000000..24ceb24f5b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/simplified.txt index f7abc0b20d..5021265f8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [excess discount amount,sum,sum(UnscaledValue(cs_ext_discount_amt))] + CometHashAggregate [sum] [excess discount amount,sum(UnscaledValue(cs_ext_discount_amt))] CometExchange #1 - CometHashAggregate [sum,cs_ext_discount_amt] + CometHashAggregate [cs_ext_discount_amt] [sum] CometProject [cs_ext_discount_amt] CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometProject [cs_ext_discount_amt,cs_sold_date_sk] @@ -11,22 +11,22 @@ WholeStageCodegen (1) CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] - CometHashAggregate [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,sum,count,avg(UnscaledValue(cs_ext_discount_amt))] + CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] CometExchange [cs_item_sk] #4 - CometHashAggregate [cs_item_sk,sum,count,cs_ext_discount_amt] + CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/explain.txt index 2f4d022c1c..107330c797 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -* ColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometHashAggregate (47) - +- CometExchange (46) - +- CometHashAggregate (45) - +- CometUnion (44) +* CometColumnarToRow (61) ++- CometTakeOrderedAndProject (60) + +- CometHashAggregate (59) + +- CometExchange (58) + +- CometHashAggregate (57) + +- CometUnion (56) :- CometHashAggregate (27) : +- CometExchange (26) : +- CometHashAggregate (25) @@ -15,23 +15,23 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) + : : +- CometNativeScan parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) + : +- CometNativeScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) : +- CometExchange (40) : +- CometHashAggregate (39) @@ -42,25 +42,44 @@ : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (28) : : : +- ReusedExchange (30) : : +- ReusedExchange (33) : +- ReusedExchange (36) - +- CometHashAggregate (43) - +- ReusedExchange (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + +- CometHashAggregate (55) + +- CometExchange (54) + +- CometHashAggregate (53) + +- CometProject (52) + +- CometBroadcastHashJoin (51) + :- CometProject (49) + : +- CometBroadcastHashJoin (48) + : :- CometProject (46) + : : +- CometBroadcastHashJoin (45) + : : :- CometFilter (43) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (42) + : : +- ReusedExchange (44) + : +- ReusedExchange (47) + +- ReusedExchange (50) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] @@ -83,9 +102,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] @@ -108,21 +130,26 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [i_item_sk#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#10, i_manufact_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometNativeScan: `spark_catalog`.`default`.`item` +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_category#12, i_manufact_id#13] -Arguments: [i_category#12, i_manufact_id#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct (18) CometFilter Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) (19) CometProject Input [2]: [i_category#12, i_manufact_id#13] @@ -164,9 +191,13 @@ Input [2]: [i_manufact_id#11, sum#14] Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(28) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (29) CometFilter Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] @@ -222,37 +253,91 @@ Input [2]: [i_manufact_id#22, sum#23] Keys [1]: [i_manufact_id#22] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] -(42) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_manufact_id#24, sum#25] - -(43) CometHashAggregate -Input [2]: [i_manufact_id#24, sum#25] -Keys [1]: [i_manufact_id#24] +(42) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(43) CometFilter +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : (isnotnull(ws_bill_addr_sk#25) AND isnotnull(ws_item_sk#24)) + +(44) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#28] + +(45) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ws_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight + +(46) CometProject +Input [5]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, d_date_sk#28] +Arguments: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26], [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] + +(47) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#29] + +(48) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] +Right output [1]: [ca_address_sk#29] +Arguments: [ws_bill_addr_sk#25], [ca_address_sk#29], Inner, BuildRight + +(49) CometProject +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ca_address_sk#29] +Arguments: [ws_item_sk#24, ws_ext_sales_price#26], [ws_item_sk#24, ws_ext_sales_price#26] + +(50) ReusedExchange [Reuses operator id: 22] +Output [2]: [i_item_sk#30, i_manufact_id#31] + +(51) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#24, ws_ext_sales_price#26] +Right output [2]: [i_item_sk#30, i_manufact_id#31] +Arguments: [ws_item_sk#24], [i_item_sk#30], Inner, BuildRight + +(52) CometProject +Input [4]: [ws_item_sk#24, ws_ext_sales_price#26, i_item_sk#30, i_manufact_id#31] +Arguments: [ws_ext_sales_price#26, i_manufact_id#31], [ws_ext_sales_price#26, i_manufact_id#31] + +(53) CometHashAggregate +Input [2]: [ws_ext_sales_price#26, i_manufact_id#31] +Keys [1]: [i_manufact_id#31] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] + +(54) CometExchange +Input [2]: [i_manufact_id#31, sum#32] +Arguments: hashpartitioning(i_manufact_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(55) CometHashAggregate +Input [2]: [i_manufact_id#31, sum#32] +Keys [1]: [i_manufact_id#31] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] -(44) CometUnion -Child 0 Input [2]: [i_manufact_id#11, total_sales#27] -Child 1 Input [2]: [i_manufact_id#22, total_sales#28] -Child 2 Input [2]: [i_manufact_id#24, total_sales#29] +(56) CometUnion +Child 0 Input [2]: [i_manufact_id#11, total_sales#33] +Child 1 Input [2]: [i_manufact_id#22, total_sales#34] +Child 2 Input [2]: [i_manufact_id#31, total_sales#35] -(45) CometHashAggregate -Input [2]: [i_manufact_id#11, total_sales#27] +(57) CometHashAggregate +Input [2]: [i_manufact_id#11, total_sales#33] Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(total_sales#27)] +Functions [1]: [partial_sum(total_sales#33)] -(46) CometExchange -Input [3]: [i_manufact_id#11, sum#30, isEmpty#31] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(58) CometExchange +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(47) CometHashAggregate -Input [3]: [i_manufact_id#11, sum#30, isEmpty#31] +(59) CometHashAggregate +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(total_sales#27)] +Functions [1]: [sum(total_sales#33)] -(48) CometTakeOrderedAndProject -Input [2]: [i_manufact_id#11, total_sales#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#32 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#32]), [i_manufact_id#11, total_sales#32], 100, [total_sales#32 ASC NULLS FIRST], [i_manufact_id#11, total_sales#32] +(60) CometTakeOrderedAndProject +Input [2]: [i_manufact_id#11, total_sales#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#38]), [i_manufact_id#11, total_sales#38], 100, 0, [total_sales#38 ASC NULLS FIRST], [i_manufact_id#11, total_sales#38] -(49) ColumnarToRow [codegen id : 1] -Input [2]: [i_manufact_id#11, total_sales#32] +(61) CometColumnarToRow [codegen id : 1] +Input [2]: [i_manufact_id#11, total_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/extended.txt new file mode 100644 index 0000000000..a210ba4b5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/extended.txt @@ -0,0 +1,89 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 86 out of 86 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/simplified.txt index dfa31808a4..5ce3cb97d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_manufact_id,total_sales] - CometHashAggregate [i_manufact_id,total_sales,sum,isEmpty,sum(total_sales)] + CometHashAggregate [sum,isEmpty] [i_manufact_id,total_sales,sum(total_sales)] CometExchange [i_manufact_id] #1 - CometHashAggregate [i_manufact_id,sum,isEmpty,total_sales] + CometHashAggregate [total_sales] [i_manufact_id,sum,isEmpty] CometUnion [i_manufact_id,total_sales] - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_manufact_id] #2 - CometHashAggregate [i_manufact_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_manufact_id,sum] CometProject [ss_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] CometProject [ss_item_sk,ss_ext_sales_price] @@ -16,26 +16,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_manufact_id] #5 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #6 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact_id] - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact_id] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_manufact_id] #7 - CometHashAggregate [i_manufact_id,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] CometProject [cs_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] CometProject [cs_item_sk,cs_ext_sales_price] @@ -43,9 +43,21 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_manufact_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] + CometProject [ws_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_manufact_id] #5 - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [i_manufact_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/explain.txt index 4d010531bc..3df476759f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,35 +15,43 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] @@ -66,9 +74,12 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#9, s_county#10] @@ -91,13 +102,16 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -134,35 +148,42 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(25) CometNativeScan: `spark_catalog`.`default`.`customer` +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (26) CometFilter Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Condition : isnotnull(c_customer_sk#17) -(27) CometBroadcastExchange +(27) CometProject Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(28) CometBroadcastHashJoin +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/simplified.txt index bf05de8c6e..66a94b088b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] @@ -16,19 +16,20 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/explain.txt index aae420dfb5..35aa0b73ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/explain.txt @@ -1,60 +1,77 @@ == Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] @@ -86,12 +103,15 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#13] @@ -105,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -118,92 +138,132 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct -(21) BroadcastHashJoin [codegen id : 5] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None -(22) Filter [codegen id : 5] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(23) Project [codegen id : 5] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_state#16] -Arguments: [ca_address_sk#15, ca_state#16] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : isnotnull(ca_address_sk#15) +(31) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] -(26) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#15, ca_state#16] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] -(27) BroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(28) BroadcastHashJoin [codegen id : 5] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(29) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#16] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#16] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(31) CometFilter -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Condition : isnotnull(cd_demo_sk#17) +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(33) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(34) BroadcastHashJoin [codegen id : 5] +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None -(35) Project [codegen id : 5] -Output [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Input [8]: [c_current_cdemo_sk#4, ca_state#16, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(36) HashAggregate [codegen id : 5] -Input [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#20), partial_max(cd_dep_count#20), partial_avg(cd_dep_count#20), partial_min(cd_dep_employed_count#21), partial_max(cd_dep_employed_count#21), partial_avg(cd_dep_employed_count#21), partial_min(cd_dep_college_count#22), partial_max(cd_dep_college_count#22), partial_avg(cd_dep_college_count#22)] -Aggregate Attributes [13]: [count#23, min#24, max#25, sum#26, count#27, min#28, max#29, sum#30, count#31, min#32, max#33, sum#34, count#35] -Results [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] - -(37) Exchange -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Arguments: hashpartitioning(ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [count(1), min(cd_dep_count#20), max(cd_dep_count#20), avg(cd_dep_count#20), min(cd_dep_employed_count#21), max(cd_dep_employed_count#21), avg(cd_dep_employed_count#21), min(cd_dep_college_count#22), max(cd_dep_college_count#22), avg(cd_dep_college_count#22)] -Aggregate Attributes [10]: [count(1)#49, min(cd_dep_count#20)#50, max(cd_dep_count#20)#51, avg(cd_dep_count#20)#52, min(cd_dep_employed_count#21)#53, max(cd_dep_employed_count#21)#54, avg(cd_dep_employed_count#21)#55, min(cd_dep_college_count#22)#56, max(cd_dep_college_count#22)#57, avg(cd_dep_college_count#22)#58] -Results [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, count(1)#49 AS cnt1#59, min(cd_dep_count#20)#50 AS min(cd_dep_count)#60, max(cd_dep_count#20)#51 AS max(cd_dep_count)#61, avg(cd_dep_count#20)#52 AS avg(cd_dep_count)#62, cd_dep_employed_count#21, count(1)#49 AS cnt2#63, min(cd_dep_employed_count#21)#53 AS min(cd_dep_employed_count)#64, max(cd_dep_employed_count#21)#54 AS max(cd_dep_employed_count)#65, avg(cd_dep_employed_count#21)#55 AS avg(cd_dep_employed_count)#66, cd_dep_college_count#22, count(1)#49 AS cnt3#67, min(cd_dep_college_count#22)#56 AS min(cd_dep_college_count)#68, max(cd_dep_college_count#22)#57 AS max(cd_dep_college_count)#69, avg(cd_dep_college_count#22)#58 AS avg(cd_dep_college_count)#70, cd_dep_count#20] - -(39) TakeOrderedAndProject -Input [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70, cd_dep_count#20] -Arguments: 100, [ca_state#16 ASC NULLS FIRST, cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_dep_count#20 ASC NULLS FIRST, cd_dep_employed_count#21 ASC NULLS FIRST, cd_dep_college_count#22 ASC NULLS FIRST], [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70] +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#23), partial_max(cd_dep_count#23), partial_avg(cd_dep_count#23), partial_min(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_avg(cd_dep_employed_count#24), partial_min(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_avg(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), min(cd_dep_count#23), max(cd_dep_count#23), avg(cd_dep_count#23), min(cd_dep_employed_count#24), max(cd_dep_employed_count#24), avg(cd_dep_employed_count#24), min(cd_dep_college_count#25), max(cd_dep_college_count#25), avg(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#23)#55, max(cd_dep_count#23)#56, avg(cd_dep_count#23)#57, min(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, avg(cd_dep_employed_count#24)#60, min(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, avg(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, count(1)#54 AS cnt1#64, min(cd_dep_count#23)#55 AS min(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#23)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#24)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#24)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, min(cd_dep_college_count#25)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#25)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#23] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#23] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/simplified.txt index 6767df5cc1..d7de10ae30 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/simplified.txt @@ -1,53 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/explain.txt index ce8b189c76..d31e815876 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/explain.txt @@ -1,46 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- Window (27) - +- * ColumnarToRow (26) - +- CometSort (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometExchange (22) - +- CometHashAggregate (21) - +- CometExpand (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (30) ++- * Project (29) + +- Window (28) + +- * CometColumnarToRow (27) + +- CometSort (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6, d_year#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#6, d_year#7] @@ -63,90 +71,100 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] -(12) CometBroadcastHashJoin +(13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11, s_state#12] - -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] - -(20) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(21) CometHashAggregate -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) + +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] + +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] + +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight + +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] + +(21) CometExpand +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] + +(22) CometHashAggregate +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(22) CometExchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#16, sum#17] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(23) CometExchange +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) CometHashAggregate -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#16, sum#17] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(24) CometHashAggregate +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -(24) CometExchange -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: hashpartitioning(_w1#21, _w2#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(25) CometExchange +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: hashpartitioning(_w1#23, _w2#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) CometSort -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22], [_w1#21 ASC NULLS FIRST, _w2#22 ASC NULLS FIRST, _w0#20 ASC NULLS FIRST] +(26) CometSort +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24], [_w1#23 ASC NULLS FIRST, _w2#24 ASC NULLS FIRST, _w0#22 ASC NULLS FIRST] -(26) ColumnarToRow [codegen id : 1] -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] +(27) CometColumnarToRow [codegen id : 1] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] -(27) Window -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: [rank(_w0#20) windowspecdefinition(_w1#21, _w2#22, _w0#20 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#23], [_w1#21, _w2#22], [_w0#20 ASC NULLS FIRST] +(28) Window +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [rank(_w0#22) windowspecdefinition(_w1#23, _w2#24, _w0#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#23, _w2#24], [_w0#22 ASC NULLS FIRST] -(28) Project [codegen id : 2] -Output [5]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] -Input [8]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22, rank_within_parent#23] +(29) Project [codegen id : 2] +Output [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Input [8]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24, rank_within_parent#25] -(29) TakeOrderedAndProject -Input [5]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] -Arguments: 100, [lochierarchy#19 DESC NULLS LAST, CASE WHEN (lochierarchy#19 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#23 ASC NULLS FIRST], [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] +(30) TakeOrderedAndProject +Input [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Arguments: 100, [lochierarchy#21 DESC NULLS LAST, CASE WHEN (lochierarchy#21 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/extended.txt new file mode 100644 index 0000000000..d49f29e436 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/extended.txt @@ -0,0 +1,32 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 26 out of 29 eligible operators (89%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/simplified.txt index 8ceb83677c..cec70668fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/simplified.txt @@ -4,13 +4,13 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] CometExchange [_w1,_w2] #1 - CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),spark_grouping_id] CometExchange [i_category,i_class,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,spark_grouping_id,sum,sum,ss_net_profit,ss_ext_sales_price] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,spark_grouping_id,sum,sum] CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] @@ -19,15 +19,16 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/explain.txt index bc7fc8f71a..4a3ef391b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (26) +* CometColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) +- CometExchange (23) @@ -13,23 +13,26 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) + : : +- CometNativeScan parquet spark_catalog.default.inventory (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) +- CometProject (19) +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (17) + +- CometNativeScan parquet spark_catalog.default.catalog_sales (17) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,694,808,940]), IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] @@ -37,101 +40,111 @@ Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) A (3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#9)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] +ReadSchema: struct (5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) (6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] (7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] (8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight (9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +(10) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] +ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11, cs_sold_date_sk#12] +(17) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (18) CometFilter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) (19) CometProject -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11], [cs_item_sk#11] +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] (20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#11] -Arguments: [i_item_sk#1], [cs_item_sk#11], Inner, BuildLeft +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/extended.txt new file mode 100644 index 0000000000..92c5233469 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.inventory + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_sales + +Comet accelerated 25 out of 25 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/simplified.txt index 2a8cf3e62b..d8d49b93e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] @@ -12,17 +12,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [cs_item_sk] CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/explain.txt index dab0b8c9ad..6841fdcbef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/explain.txt @@ -1,44 +1,72 @@ == Physical Plan == -* ColumnarToRow (27) -+- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometBroadcastHashJoin (20) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (48) ++- CometHashAggregate (47) + +- CometExchange (46) + +- CometHashAggregate (45) + +- CometProject (44) + +- CometBroadcastHashJoin (43) + :- CometBroadcastHashJoin (30) + : :- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- CometBroadcastExchange (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - +- ReusedExchange (21) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : +- CometBroadcastExchange (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- CometBroadcastExchange (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (31) + : +- ReusedExchange (33) + +- ReusedExchange (36) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] @@ -61,83 +89,183 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(9) CometNativeScan: `spark_catalog`.`default`.`customer` +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] + +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(18) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) + +(20) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] + +(21) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(22) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] + +(23) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] + +(24) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(25) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] + +(26) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(27) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(28) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] +(29) CometBroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] + +(30) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight + +(31) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(32) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) + +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] + +(34) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(18) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] +(35) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] + +(36) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] + +(37) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight + +(38) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] + +(39) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -(19) CometBroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [c_last_name#9, c_first_name#10, d_date#11] +(40) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(20) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)], LeftSemi, BuildRight +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] -(21) ReusedExchange [Reuses operator id: 19] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] +(42) CometBroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] -(22) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#12, c_first_name#13, d_date#14] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight +(43) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight -(23) CometProject -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(44) CometProject +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(24) CometHashAggregate +(45) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -(25) CometExchange -Input [1]: [count#15] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(46) CometExchange +Input [1]: [count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(26) CometHashAggregate -Input [1]: [count#15] +(47) CometHashAggregate +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] -(27) ColumnarToRow [codegen id : 1] -Input [1]: [count(1)#16] +(48) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/extended.txt new file mode 100644 index 0000000000..ab9f89f4f0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/extended.txt @@ -0,0 +1,62 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 59 out of 59 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/simplified.txt index c2697a2c1c..b912edf4d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/simplified.txt @@ -1,7 +1,7 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [count(1),count,count(1)] + CometHashAggregate [count] [count(1),count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -15,15 +15,36 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - ReusedExchange [c_last_name,c_first_name,d_date] #5 + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/explain.txt index f1914bd440..e0b2fab87d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -* ColumnarToRow (47) +* CometColumnarToRow (47) +- CometSort (46) - +- CometColumnarExchange (45) + +- CometExchange (45) +- CometBroadcastHashJoin (44) :- CometProject (23) : +- CometFilter (22) @@ -15,17 +15,17 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -39,26 +39,33 @@ : :- CometProject (28) : : +- CometBroadcastHashJoin (27) : : :- CometFilter (25) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (24) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (26) : +- ReusedExchange (29) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (32) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [i_item_sk#5] @@ -77,9 +84,12 @@ Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -98,9 +108,12 @@ Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] @@ -145,9 +158,13 @@ Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17) Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] -(24) CometNativeScan: `spark_catalog`.`default`.`inventory` +(24) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Arguments: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#22)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (25) CometFilter Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] @@ -177,9 +194,12 @@ Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -(32) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(32) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#26, d_year#27, d_moy#28] -Arguments: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct (33) CometFilter Input [3]: [d_date_sk#26, d_year#27, d_moy#28] @@ -233,14 +253,14 @@ Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight -(45) CometColumnarExchange +(45) CometExchange Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] -(47) ColumnarToRow [codegen id : 1] +(47) CometColumnarToRow [codegen id : 1] Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/extended.txt new file mode 100644 index 0000000000..dab448e5d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/simplified.txt index 39d6ad8918..cdbd0d9063 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] @@ -16,23 +16,23 @@ WholeStageCodegen (1) CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] @@ -40,10 +40,10 @@ WholeStageCodegen (1) CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] ReusedExchange [i_item_sk] #3 ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/explain.txt index 90780b4060..62b5730314 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -* ColumnarToRow (47) +* CometColumnarToRow (47) +- CometSort (46) - +- CometColumnarExchange (45) + +- CometExchange (45) +- CometBroadcastHashJoin (44) :- CometProject (23) : +- CometFilter (22) @@ -15,17 +15,17 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -39,26 +39,33 @@ : :- CometProject (28) : : +- CometBroadcastHashJoin (27) : : :- CometFilter (25) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (24) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (26) : +- ReusedExchange (29) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (32) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [i_item_sk#5] @@ -77,9 +84,12 @@ Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -98,9 +108,12 @@ Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] @@ -145,9 +158,13 @@ Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17 Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] -(24) CometNativeScan: `spark_catalog`.`default`.`inventory` +(24) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Arguments: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#22)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (25) CometFilter Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] @@ -177,9 +194,12 @@ Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -(32) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(32) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#26, d_year#27, d_moy#28] -Arguments: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct (33) CometFilter Input [3]: [d_date_sk#26, d_year#27, d_moy#28] @@ -233,14 +253,14 @@ Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight -(45) CometColumnarExchange +(45) CometExchange Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] -(47) ColumnarToRow [codegen id : 1] +(47) CometColumnarToRow [codegen id : 1] Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/extended.txt new file mode 100644 index 0000000000..dab448e5d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/simplified.txt index 39d6ad8918..cdbd0d9063 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] @@ -16,23 +16,23 @@ WholeStageCodegen (1) CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] @@ -40,10 +40,10 @@ WholeStageCodegen (1) CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] ReusedExchange [i_item_sk] #3 ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/explain.txt index 96845f708b..bb574f8279 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/explain.txt @@ -1,262 +1,565 @@ == Physical Plan == -* ColumnarToRow (50) -+- CometTakeOrderedAndProject (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (45) - : +- CometBroadcastHashJoin (44) - : :- CometProject (42) - : : +- CometBroadcastHashJoin (41) - : : :- CometProject (37) - : : : +- CometBroadcastHashJoin (36) - : : : :- CometBroadcastHashJoin (31) - : : : : :- CometFilter (16) - : : : : : +- CometHashAggregate (15) - : : : : : +- CometExchange (14) - : : : : : +- CometHashAggregate (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : : +- CometBroadcastExchange (30) - : : : : +- CometHashAggregate (29) - : : : : +- CometExchange (28) - : : : : +- CometHashAggregate (27) - : : : : +- CometProject (26) - : : : : +- CometBroadcastHashJoin (25) - : : : : :- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometFilter (18) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : : : +- ReusedExchange (19) - : : : : +- CometBroadcastExchange (24) - : : : : +- CometFilter (23) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : : : +- CometBroadcastExchange (35) - : : : +- CometFilter (34) - : : : +- CometHashAggregate (33) - : : : +- ReusedExchange (32) - : : +- CometBroadcastExchange (40) - : : +- CometHashAggregate (39) - : : +- ReusedExchange (38) - : +- ReusedExchange (43) - +- ReusedExchange (46) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) + : : : : :- CometFilter (17) + : : : : : +- CometHashAggregate (16) + : : : : : +- CometExchange (15) + : : : : : +- CometHashAggregate (14) + : : : : : +- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometProject (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometNativeScan parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometNativeScan parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(4) CometFilter -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(9) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(11) CometBroadcastHashJoin -Left output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Right output [2]: [d_date_sk#15, d_year#16] -Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] - -(13) CometHashAggregate -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] - -(14) CometExchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#17, isEmpty#18] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#17, isEmpty#18] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] - -(16) CometFilter -Input [2]: [customer_id#19, year_total#20] -Condition : (isnotnull(year_total#20) AND (year_total#20 > 0.000000)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Arguments: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] - -(18) CometFilter -Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [6]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] - -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Right output [6]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Arguments: [c_customer_sk#21], [ss_customer_sk#29], Inner, BuildRight - -(21) CometProject -Input [14]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#35, d_year#36] -Arguments: [d_date_sk#35, d_year#36] - -(23) CometFilter -Input [2]: [d_date_sk#35, d_year#36] -Condition : ((isnotnull(d_year#36) AND (d_year#36 = 2002)) AND isnotnull(d_date_sk#35)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#35, d_year#36] -Arguments: [d_date_sk#35, d_year#36] - -(25) CometBroadcastHashJoin -Left output [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Right output [2]: [d_date_sk#35, d_year#36] -Arguments: [ss_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight - -(26) CometProject -Input [14]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34, d_date_sk#35, d_year#36] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36] - -(27) CometHashAggregate -Input [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36] -Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36] -Functions [1]: [partial_sum(((((ss_ext_list_price#33 - ss_ext_wholesale_cost#32) - ss_ext_discount_amt#30) + ss_ext_sales_price#31) / 2))] - -(28) CometExchange -Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, sum#37, isEmpty#38] -Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(29) CometHashAggregate -Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, sum#37, isEmpty#38] -Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36] -Functions [1]: [sum(((((ss_ext_list_price#33 - ss_ext_wholesale_cost#32) - ss_ext_discount_amt#30) + ss_ext_sales_price#31) / 2))] - -(30) CometBroadcastExchange -Input [8]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#19, year_total#20] -Right output [8]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Arguments: [customer_id#19], [customer_id#39], Inner, BuildRight - -(32) ReusedExchange [Reuses operator id: 14] -Output [10]: [c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53, d_year#54, sum#55, isEmpty#56] - -(33) CometHashAggregate -Input [10]: [c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53, d_year#54, sum#55, isEmpty#56] -Keys [8]: [c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53, d_year#54] -Functions [1]: [sum(((((cs_ext_list_price#57 - cs_ext_wholesale_cost#58) - cs_ext_discount_amt#59) + cs_ext_sales_price#60) / 2))] - -(34) CometFilter -Input [2]: [customer_id#61, year_total#62] -Condition : (isnotnull(year_total#62) AND (year_total#62 > 0.000000)) - -(35) CometBroadcastExchange -Input [2]: [customer_id#61, year_total#62] -Arguments: [customer_id#61, year_total#62] - -(36) CometBroadcastHashJoin -Left output [10]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Right output [2]: [customer_id#61, year_total#62] -Arguments: [customer_id#19], [customer_id#61], Inner, BuildRight - -(37) CometProject -Input [12]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, customer_id#61, year_total#62] -Arguments: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62], [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62] - -(38) ReusedExchange [Reuses operator id: 28] -Output [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#70, sum#71, isEmpty#72] +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] + +(12) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] + +(14) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(15) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(17) CometFilter +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true))) + +(20) CometProject +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#29, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#30, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#31, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#33, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#34, 50, true, false, true) AS c_email_address#14] + +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] + +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_sk#27], [ss_customer_sk#35], Inner, BuildRight + +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#41, d_year#42] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#41, d_year#42] +Condition : ((isnotnull(d_year#42) AND (d_year#42 = 2002)) AND isnotnull(d_date_sk#41)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#41, d_year#42] +Arguments: [d_date_sk#41, d_year#42] + +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Right output [2]: [d_date_sk#41, d_year#42] +Arguments: [ss_sold_date_sk#40], [d_date_sk#41], Inner, BuildRight + +(28) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40, d_date_sk#41, d_year#42] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] -(39) CometHashAggregate -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#70, sum#71, isEmpty#72] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#70] -Functions [1]: [sum(((((cs_ext_list_price#73 - cs_ext_wholesale_cost#74) - cs_ext_discount_amt#75) + cs_ext_sales_price#76) / 2))] +(29) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [partial_sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(30) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(32) CometBroadcastExchange +Input [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#25], [customer_id#45], Inner, BuildRight + +(34) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(35) CometFilter +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Condition : (isnotnull(c_customer_sk#53) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true))) + +(36) CometProject +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#55, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#56, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#57, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#59, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#60, 50, true, false, true) AS c_email_address#66] + +(37) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#72)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(38) CometFilter +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Condition : isnotnull(cs_bill_customer_sk#67) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_sk#53], [cs_bill_customer_sk#67], Inner, BuildRight + +(41) CometProject +Input [14]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#73, d_year#74] + +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight + +(44) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72, d_date_sk#73, d_year#74] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] + +(45) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [partial_sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(46) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(48) CometFilter +Input [2]: [customer_id#77, year_total#78] +Condition : (isnotnull(year_total#78) AND (year_total#78 > 0.000000)) -(40) CometBroadcastExchange +(49) CometBroadcastExchange Input [2]: [customer_id#77, year_total#78] Arguments: [customer_id#77, year_total#78] -(41) CometBroadcastHashJoin -Left output [11]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62] +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] Right output [2]: [customer_id#77, year_total#78] -Arguments: [customer_id#19], [customer_id#77], Inner, (CASE WHEN (year_total#62 > 0.000000) THEN (year_total#78 / year_total#62) END > CASE WHEN (year_total#20 > 0.000000) THEN (year_total#46 / year_total#20) END), BuildRight - -(42) CometProject -Input [13]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62, customer_id#77, year_total#78] -Arguments: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78], [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78] - -(43) ReusedExchange [Reuses operator id: 35] -Output [2]: [customer_id#79, year_total#80] - -(44) CometBroadcastHashJoin -Left output [10]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78] -Right output [2]: [customer_id#79, year_total#80] -Arguments: [customer_id#19], [customer_id#79], Inner, BuildRight - -(45) CometProject -Input [12]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, customer_id#79, year_total#80] -Arguments: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80], [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80] - -(46) ReusedExchange [Reuses operator id: 40] -Output [2]: [customer_id#81, year_total#82] - -(47) CometBroadcastHashJoin -Left output [11]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80] -Right output [2]: [customer_id#81, year_total#82] -Arguments: [customer_id#19], [customer_id#81], Inner, (CASE WHEN (year_total#62 > 0.000000) THEN (year_total#78 / year_total#62) END > CASE WHEN (year_total#80 > 0.000000) THEN (year_total#82 / year_total#80) END), BuildRight - -(48) CometProject -Input [13]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80, customer_id#81, year_total#82] -Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] - -(49) CometTakeOrderedAndProject -Input [7]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_preferred_cust_flag#42 ASC NULLS FIRST,customer_birth_country#43 ASC NULLS FIRST,customer_login#44 ASC NULLS FIRST,customer_email_address#45 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_preferred_cust_flag#42,customer_birth_country#43,customer_login#44,customer_email_address#45]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45], 100, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_preferred_cust_flag#42 ASC NULLS FIRST, customer_birth_country#43 ASC NULLS FIRST, customer_login#44 ASC NULLS FIRST, customer_email_address#45 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] - -(50) ColumnarToRow [codegen id : 1] -Input [7]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] +Arguments: [customer_id#25], [customer_id#77], Inner, BuildRight + +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, customer_id#77, year_total#78] +Arguments: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78], [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(53) CometFilter +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Condition : (isnotnull(c_customer_sk#79) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true))) + +(54) CometProject +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Arguments: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66], [c_customer_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#81, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#82, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#83, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#85, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#86, 50, true, false, true) AS c_email_address#66] + +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] + +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_sk#79], [cs_bill_customer_sk#87], Inner, BuildRight + +(57) CometProject +Input [14]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#93, d_year#94] + +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [cs_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight + +(60) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] + +(61) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [partial_sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] + +(62) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#97, year_total#98] + +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] +Right output [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#25], [customer_id#97], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#52 / year_total#26) END), BuildRight + +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78, customer_id#97, year_total#98] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] + +(67) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(68) CometFilter +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Condition : (isnotnull(c_customer_sk#99) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true))) + +(69) CometProject +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Arguments: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112], [c_customer_sk#99, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#101, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#102, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#103, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#104, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#105, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#106, 50, true, false, true) AS c_email_address#112] + +(70) CometNativeScan parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#118)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(71) CometFilter +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Condition : isnotnull(ws_bill_customer_sk#113) + +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] + +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_sk#99], [ws_bill_customer_sk#113], Inner, BuildRight + +(74) CometProject +Input [14]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] + +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#119, d_year#120] + +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Right output [2]: [d_date_sk#119, d_year#120] +Arguments: [ws_sold_date_sk#118], [d_date_sk#119], Inner, BuildRight + +(77) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118, d_date_sk#119, d_year#120] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] + +(78) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [partial_sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] + +(79) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(80) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] + +(81) CometFilter +Input [2]: [customer_id#123, year_total#124] +Condition : (isnotnull(year_total#124) AND (year_total#124 > 0.000000)) + +(82) CometBroadcastExchange +Input [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#123, year_total#124] + +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] +Right output [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#25], [customer_id#123], Inner, BuildRight + +(84) CometProject +Input [12]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, customer_id#123, year_total#124] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] + +(85) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(86) CometFilter +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Condition : (isnotnull(c_customer_sk#125) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true))) + +(87) CometProject +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Arguments: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112], [c_customer_sk#125, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#127, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#128, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#129, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#131, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#132, 50, true, false, true) AS c_email_address#112] + +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] + +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_sk#125], [ws_bill_customer_sk#133], Inner, BuildRight + +(90) CometProject +Input [14]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] + +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#139, d_year#140] + +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Right output [2]: [d_date_sk#139, d_year#140] +Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight + +(93) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138, d_date_sk#139, d_year#140] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] + +(94) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [partial_sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(95) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(96) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(97) CometBroadcastExchange +Input [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#143, year_total#144] + +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] +Right output [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#25], [customer_id#143], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#124 > 0.000000) THEN (year_total#144 / year_total#124) END), BuildRight + +(99) CometProject +Input [13]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124, customer_id#143, year_total#144] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#45 ASC NULLS FIRST,customer_first_name#46 ASC NULLS FIRST,customer_last_name#47 ASC NULLS FIRST,customer_preferred_cust_flag#48 ASC NULLS FIRST,customer_birth_country#49 ASC NULLS FIRST,customer_login#50 ASC NULLS FIRST,customer_email_address#51 ASC NULLS FIRST], output=[customer_id#45,customer_first_name#46,customer_last_name#47,customer_preferred_cust_flag#48,customer_birth_country#49,customer_login#50,customer_email_address#51]), [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], 100, 0, [customer_id#45 ASC NULLS FIRST, customer_first_name#46 ASC NULLS FIRST, customer_last_name#47 ASC NULLS FIRST, customer_preferred_cust_flag#48 ASC NULLS FIRST, customer_birth_country#49 ASC NULLS FIRST, customer_login#50 ASC NULLS FIRST, customer_email_address#51 ASC NULLS FIRST], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/extended.txt new file mode 100644 index 0000000000..7b0b7d1cc0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/extended.txt @@ -0,0 +1,117 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 114 out of 114 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/simplified.txt index 7754033520..f837111fe3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] CometProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] @@ -12,41 +12,92 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #1 - CometBroadcastExchange [customer_id,year_total] #8 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #5 - ReusedExchange [customer_id,year_total] #7 - ReusedExchange [customer_id,year_total] #8 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/explain.txt index d78d968483..572c9d73af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/explain.txt @@ -1,41 +1,46 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometTakeOrderedAndProject (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) : : :- CometProject (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (5) - : : +- CometBroadcastExchange (14) - : : +- CometFilter (13) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (12) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - +- CometBroadcastExchange (25) - +- CometFilter (24) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (26) + +- CometFilter (25) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] @@ -49,9 +54,12 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(5) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct (6) CometFilter Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] @@ -78,91 +86,104 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -(12) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(12) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (13) CometFilter Input [2]: [w_warehouse_sk#10, w_state#11] Condition : isnotnull(w_warehouse_sk#10) -(14) CometBroadcastExchange +(14) CometProject Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#11, 2, true, false, true) AS w_state#12] -(15) CometBroadcastHashJoin +(15) CometBroadcastExchange +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] + +(16) CometBroadcastHashJoin Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -Right output [2]: [w_warehouse_sk#10, w_state#11] +Right output [2]: [w_warehouse_sk#10, w_state#12] Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight -(16) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(18) CometFilter -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) - -(19) CometProject -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13], [i_item_sk#12, i_item_id#13] - -(20) CometBroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: [i_item_sk#12, i_item_id#13] - -(21) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Right output [2]: [i_item_sk#12, i_item_id#13] -Arguments: [cs_item_sk#2], [i_item_sk#12], Inner, BuildRight - -(22) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(24) CometFilter -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(26) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#5], [d_date_sk#15], Inner, BuildRight - -(27) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] -Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16], [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] - -(28) CometHashAggregate -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] - -(29) CometExchange -Input [6]: [w_state#11, i_item_id#13, sum#17, isEmpty#18, sum#19, isEmpty#20] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(30) CometHashAggregate -Input [6]: [w_state#11, i_item_id#13, sum#17, isEmpty#18, sum#19, isEmpty#20] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] - -(31) CometTakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#21, sales_after#22] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#11 ASC NULLS FIRST,i_item_id#13 ASC NULLS FIRST], output=[w_state#11,i_item_id#13,sales_before#21,sales_after#22]), [w_state#11, i_item_id#13, sales_before#21, sales_after#22], 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#21, sales_after#22] - -(32) ColumnarToRow [codegen id : 1] -Input [4]: [w_state#11, i_item_id#13, sales_before#21, sales_after#22] +(17) CometProject +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] + +(18) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] +ReadSchema: struct + +(19) CometFilter +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) + +(20) CometProject +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#14, 16, true, false, true) AS i_item_id#16] + +(21) CometBroadcastExchange +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] + +(22) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight + +(23) CometProject +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#17, d_date#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] + +(27) CometBroadcastHashJoin +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight + +(28) CometProject +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] + +(29) CometHashAggregate +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(30) CometExchange +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(31) CometHashAggregate +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(32) CometTakeOrderedAndProject +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] + +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/extended.txt new file mode 100644 index 0000000000..7a22197a97 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/simplified.txt index e2f68d5b37..98932cc775 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - CometHashAggregate [w_state,i_item_id,sales_before,sales_after,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [w_state,i_item_id,sales_before,sales_after,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] CometExchange [w_state,i_item_id] #1 - CometHashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty,d_date,cs_sales_price,cr_refunded_cash] + CometHashAggregate [d_date,cs_sales_price,cr_refunded_cash] [w_state,i_item_id,sum,isEmpty,sum,isEmpty] CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] @@ -16,19 +16,20 @@ WholeStageCodegen (1) CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] CometExchange [cr_order_number,cr_item_sk] #3 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometBroadcastExchange [w_warehouse_sk,w_state] #4 - CometFilter [w_warehouse_sk,w_state] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_state] + CometProject [w_state] [w_warehouse_sk,w_state] + CometFilter [w_warehouse_sk,w_state] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometProject [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #6 CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/explain.txt index e1a62cf025..625df9cd57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : +- CometNativeScan parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,86 +17,91 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometNativeScan: `spark_catalog`.`default`.`item` (4) + +- CometNativeScan parquet spark_catalog.default.item (4) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Arguments: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778)] +ReadSchema: struct (2) CometFilter Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) +Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) (3) CometProject Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometNativeScan: `spark_catalog`.`default`.`item` +(4) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct (5) CometFilter Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) +Condition : (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = powder ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = khaki )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Ounce ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Oz ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = brown ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = honeydew )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Bunch ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Ton ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = floral ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = deep )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Dozen ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = light ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = cornflower )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Box ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Pound ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large )))))) OR (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = midnight ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = snow )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Pallet ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Gross ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = cyan ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = papaya )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Cup ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Dram ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = orange ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = frosted )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Each ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Tbl ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = forest ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = ghost )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Lb ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Bundle ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large ))))))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#5, 50, true, false, true))) (6) CometProject Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_manufact#5], [i_manufact#5] +Arguments: [i_manufact#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#5, 50, true, false, true) AS i_manufact#9] (7) CometHashAggregate -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] +Input [1]: [i_manufact#9] +Keys [1]: [i_manufact#9] Functions [1]: [partial_count(1)] (8) CometExchange -Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_manufact#9, count#10] +Arguments: hashpartitioning(i_manufact#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (9) CometHashAggregate -Input [2]: [i_manufact#5, count#9] -Keys [1]: [i_manufact#5] +Input [2]: [i_manufact#9, count#10] +Keys [1]: [i_manufact#9] Functions [1]: [count(1)] (10) CometFilter -Input [2]: [item_cnt#10, i_manufact#5] -Condition : (item_cnt#10 > 0) +Input [2]: [item_cnt#11, i_manufact#9] +Condition : (item_cnt#11 > 0) (11) CometProject -Input [2]: [item_cnt#10, i_manufact#5] -Arguments: [i_manufact#5], [i_manufact#5] +Input [2]: [item_cnt#11, i_manufact#9] +Arguments: [i_manufact#9], [i_manufact#9] (12) CometBroadcastExchange -Input [1]: [i_manufact#5] -Arguments: [i_manufact#5] +Input [1]: [i_manufact#9] +Arguments: [i_manufact#9] (13) CometBroadcastHashJoin Left output [2]: [i_manufact#2, i_product_name#3] -Right output [1]: [i_manufact#5] -Arguments: [i_manufact#2], [i_manufact#5], Inner, BuildRight +Right output [1]: [i_manufact#9] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true)], [i_manufact#9], Inner, BuildRight (14) CometProject -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] -Arguments: [i_product_name#3], [i_product_name#3] +Input [3]: [i_manufact#2, i_product_name#3, i_manufact#9] +Arguments: [i_product_name#12], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#3, 50, true, false, true) AS i_product_name#12] (15) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] Functions: [] (16) CometExchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [1]: [i_product_name#12] +Arguments: hashpartitioning(i_product_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (17) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] Functions: [] (18) CometTakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#3 ASC NULLS FIRST], output=[i_product_name#3]), [i_product_name#3], 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] +Input [1]: [i_product_name#12] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#12 ASC NULLS FIRST], output=[i_product_name#12]), [i_product_name#12], 100, 0, [i_product_name#12 ASC NULLS FIRST], [i_product_name#12] -(19) ColumnarToRow [codegen id : 1] -Input [1]: [i_product_name#3] +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [i_product_name#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/extended.txt new file mode 100644 index 0000000000..b40ebddb6f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/simplified.txt index ea03f38e2c..2557f51684 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_product_name] CometHashAggregate [i_product_name] CometExchange [i_product_name] #1 CometHashAggregate [i_product_name] - CometProject [i_product_name] + CometProject [i_product_name] [i_product_name] CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_manufact_id,i_manufact,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] - CometHashAggregate [item_cnt,i_manufact,count,count(1)] + CometHashAggregate [count] [item_cnt,i_manufact,count(1)] CometExchange [i_manufact] #3 CometHashAggregate [i_manufact,count] - CometProject [i_manufact] + CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact,i_size,i_color,i_units] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/explain.txt index ed78e7e69c..d0f741f459 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,19 +10,22 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] @@ -63,39 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnul (11) CometProject Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9], [i_item_sk#7, i_category_id#8, i_category#9] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11], [i_item_sk#7, i_category_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9] +Input [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_category_id#8, i_category#9] +Right output [3]: [i_item_sk#7, i_category_id#8, i_category#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] (15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sum(ss_ext_sales_price)#12 DESC NULLS LAST,d_year#2 ASC NULLS FIRST,i_category_id#8 ASC NULLS FIRST,i_category#9 ASC NULLS FIRST], output=[d_year#2,i_category_id#8,i_category#9,sum(ss_ext_sales_price)#12]), [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12], 100, [sum(ss_ext_sales_price)#12 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sum(ss_ext_sales_price)#13 DESC NULLS LAST,d_year#2 ASC NULLS FIRST,i_category_id#8 ASC NULLS FIRST,i_category#11 ASC NULLS FIRST], output=[d_year#2,i_category_id#8,i_category#11,sum(ss_ext_sales_price)#13]), [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13], 100, 0, [sum(ss_ext_sales_price)#13 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#11 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/simplified.txt index 12bf8b222e..96ab34454d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [d_year,i_category_id,i_category,sum(ss_ext_sales_price)] - CometHashAggregate [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum(UnscaledValue(ss_ext_sales_price))] CometExchange [d_year,i_category_id,i_category] #1 - CometHashAggregate [d_year,i_category_id,i_category,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [d_year,i_category_id,i_category,sum] CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] CometProject [d_year,ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 - CometProject [i_item_sk,i_category_id,i_category] + CometProject [i_category] [i_item_sk,i_category_id,i_category] CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category_id,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/explain.txt index a5550e3d37..807307f250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,19 +10,22 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + +- CometNativeScan parquet spark_catalog.default.store (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_year#2, d_day_name#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] @@ -30,72 +33,79 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk (3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_day_name#3], [d_date_sk#1, d_day_name#3] +Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_store_sk#5) (6) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] (7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_day_name#3] -Right output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight +Left output [2]: [d_date_sk#1, d_day_name#4] +Right output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight (8) CometProject -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_day_name#3, ss_store_sk#4, ss_sales_price#5], [d_day_name#3, ss_store_sk#4, ss_sales_price#5] +Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] +(9) CometNativeScan parquet spark_catalog.default.store +Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8)) (11) CometProject -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9], [s_store_sk#7, s_store_id#8, s_store_name#9] +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10], [s_store_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#9, 16, true, false, true) AS s_store_id#12, s_store_name#10] (12) CometBroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9] +Input [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10] (13) CometBroadcastHashJoin -Left output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Right output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [ss_store_sk#4], [s_store_sk#7], Inner, BuildRight +Left output [3]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6] +Right output [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [ss_store_sk#5], [s_store_sk#8], Inner, BuildRight (14) CometProject -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9], [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] +Input [6]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6, s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10], [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] (15) CometHashAggregate -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] +Input [4]: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] (16) CometExchange -Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(s_store_name#10, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] (18) CometTakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#9 ASC NULLS FIRST,s_store_id#8 ASC NULLS FIRST,sun_sales#18 ASC NULLS FIRST,mon_sales#19 ASC NULLS FIRST,tue_sales#20 ASC NULLS FIRST,wed_sales#21 ASC NULLS FIRST,thu_sales#22 ASC NULLS FIRST,fri_sales#23 ASC NULLS FIRST,sat_sales#24 ASC NULLS FIRST], output=[s_store_name#9,s_store_id#8,sun_sales#18,mon_sales#19,tue_sales#20,wed_sales#21,thu_sales#22,fri_sales#23,sat_sales#24]), [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24], 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#18 ASC NULLS FIRST, mon_sales#19 ASC NULLS FIRST, tue_sales#20 ASC NULLS FIRST, wed_sales#21 ASC NULLS FIRST, thu_sales#22 ASC NULLS FIRST, fri_sales#23 ASC NULLS FIRST, sat_sales#24 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#10 ASC NULLS FIRST,s_store_id#12 ASC NULLS FIRST,sun_sales#20 ASC NULLS FIRST,mon_sales#21 ASC NULLS FIRST,tue_sales#22 ASC NULLS FIRST,wed_sales#23 ASC NULLS FIRST,thu_sales#24 ASC NULLS FIRST,fri_sales#25 ASC NULLS FIRST,sat_sales#26 ASC NULLS FIRST], output=[s_store_name#10,s_store_id#12,sun_sales#20,mon_sales#21,tue_sales#22,wed_sales#23,thu_sales#24,fri_sales#25,sat_sales#26]), [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26], 100, 0, [s_store_name#10 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sun_sales#20 ASC NULLS FIRST, mon_sales#21 ASC NULLS FIRST, tue_sales#22 ASC NULLS FIRST, wed_sales#23 ASC NULLS FIRST, thu_sales#24 ASC NULLS FIRST, fri_sales#25 ASC NULLS FIRST, sat_sales#26 ASC NULLS FIRST], [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] -(19) ColumnarToRow [codegen id : 1] -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] +(19) CometColumnarToRow [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/extended.txt new file mode 100644 index 0000000000..5eaea983af --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/simplified.txt index bb39c4233c..c7064b3ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - CometHashAggregate [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] CometExchange [s_store_name,s_store_id] #1 - CometHashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] + CometHashAggregate [d_day_name,ss_sales_price] [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] CometProject [d_day_name,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_day_name] + CometProject [d_day_name] [d_date_sk,d_day_name] CometFilter [d_date_sk,d_year,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometProject [s_store_sk,s_store_id,s_store_name] + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/explain.txt index 3399127da3..b2a660d9d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/explain.txt @@ -1,51 +1,57 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * SortMergeJoin Inner (31) - : : :- * Sort (17) - : : : +- * Project (16) - : : : +- * Filter (15) - : : : +- Window (14) - : : : +- WindowGroupLimit (13) - : : : +- * Sort (12) - : : : +- Exchange (11) - : : : +- WindowGroupLimit (10) - : : : +- * ColumnarToRow (9) - : : : +- CometSort (8) - : : : +- CometFilter (7) - : : : +- CometHashAggregate (6) - : : : +- CometExchange (5) - : : : +- CometHashAggregate (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- * Sort (30) - : : +- * Project (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- WindowGroupLimit (26) - : : +- * Sort (25) - : : +- Exchange (24) - : : +- WindowGroupLimit (23) - : : +- * ColumnarToRow (22) - : : +- CometSort (21) - : : +- CometFilter (20) - : : +- CometHashAggregate (19) - : : +- ReusedExchange (18) - : +- BroadcastExchange (36) - : +- * ColumnarToRow (35) - : +- CometFilter (34) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (33) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (34) + : : +- * SortMergeJoin Inner (33) + : : :- * Sort (18) + : : : +- * Project (17) + : : : +- * Filter (16) + : : : +- Window (15) + : : : +- WindowGroupLimit (14) + : : : +- * CometColumnarToRow (13) + : : : +- CometSort (12) + : : : +- CometColumnarExchange (11) + : : : +- WindowGroupLimit (10) + : : : +- * CometColumnarToRow (9) + : : : +- CometSort (8) + : : : +- CometFilter (7) + : : : +- CometHashAggregate (6) + : : : +- CometExchange (5) + : : : +- CometHashAggregate (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- * Sort (32) + : : +- * Project (31) + : : +- * Filter (30) + : : +- Window (29) + : : +- WindowGroupLimit (28) + : : +- * CometColumnarToRow (27) + : : +- CometSort (26) + : : +- CometColumnarExchange (25) + : : +- WindowGroupLimit (24) + : : +- * CometColumnarToRow (23) + : : +- CometSort (22) + : : +- CometFilter (21) + : : +- CometHashAggregate (20) + : : +- ReusedExchange (19) + : +- BroadcastExchange (39) + : +- * CometColumnarToRow (38) + : +- CometProject (37) + : +- CometFilter (36) + : +- CometNativeScan parquet spark_catalog.default.item (35) + +- ReusedExchange (42) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] @@ -77,185 +83,201 @@ Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 Input [2]: [item_sk#7, rank_col#8] Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] -(9) ColumnarToRow [codegen id : 1] +(9) CometColumnarToRow [codegen id : 1] Input [2]: [item_sk#7, rank_col#8] (10) WindowGroupLimit Input [2]: [item_sk#7, rank_col#8] Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Partial -(11) Exchange +(11) CometColumnarExchange Input [2]: [item_sk#7, rank_col#8] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(12) Sort [codegen id : 2] +(12) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(13) CometColumnarToRow [codegen id : 2] Input [2]: [item_sk#7, rank_col#8] -Arguments: [rank_col#8 ASC NULLS FIRST], false, 0 -(13) WindowGroupLimit +(14) WindowGroupLimit Input [2]: [item_sk#7, rank_col#8] Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Final -(14) Window +(15) Window Input [2]: [item_sk#7, rank_col#8] Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] -(15) Filter [codegen id : 3] +(16) Filter [codegen id : 3] Input [3]: [item_sk#7, rank_col#8, rnk#11] Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) -(16) Project [codegen id : 3] +(17) Project [codegen id : 3] Output [2]: [item_sk#7, rnk#11] Input [3]: [item_sk#7, rank_col#8, rnk#11] -(17) Sort [codegen id : 3] +(18) Sort [codegen id : 3] Input [2]: [item_sk#7, rnk#11] Arguments: [rnk#11 ASC NULLS FIRST], false, 0 -(18) ReusedExchange [Reuses operator id: 5] +(19) ReusedExchange [Reuses operator id: 5] Output [3]: [ss_item_sk#12, sum#13, count#14] -(19) CometHashAggregate +(20) CometHashAggregate Input [3]: [ss_item_sk#12, sum#13, count#14] Keys [1]: [ss_item_sk#12] Functions [1]: [avg(UnscaledValue(ss_net_profit#15))] -(20) CometFilter +(21) CometFilter Input [2]: [item_sk#16, rank_col#17] Condition : (isnotnull(rank_col#17) AND (cast(rank_col#17 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) -(21) CometSort +(22) CometSort Input [2]: [item_sk#16, rank_col#17] Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] -(22) ColumnarToRow [codegen id : 4] +(23) CometColumnarToRow [codegen id : 4] Input [2]: [item_sk#16, rank_col#17] -(23) WindowGroupLimit +(24) WindowGroupLimit Input [2]: [item_sk#16, rank_col#17] Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Partial -(24) Exchange +(25) CometColumnarExchange Input [2]: [item_sk#16, rank_col#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(26) CometSort +Input [2]: [item_sk#16, rank_col#17] +Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] -(25) Sort [codegen id : 5] +(27) CometColumnarToRow [codegen id : 5] Input [2]: [item_sk#16, rank_col#17] -Arguments: [rank_col#17 DESC NULLS LAST], false, 0 -(26) WindowGroupLimit +(28) WindowGroupLimit Input [2]: [item_sk#16, rank_col#17] Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Final -(27) Window +(29) Window Input [2]: [item_sk#16, rank_col#17] Arguments: [rank(rank_col#17) windowspecdefinition(rank_col#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#18], [rank_col#17 DESC NULLS LAST] -(28) Filter [codegen id : 6] +(30) Filter [codegen id : 6] Input [3]: [item_sk#16, rank_col#17, rnk#18] Condition : ((rnk#18 < 11) AND isnotnull(item_sk#16)) -(29) Project [codegen id : 6] +(31) Project [codegen id : 6] Output [2]: [item_sk#16, rnk#18] Input [3]: [item_sk#16, rank_col#17, rnk#18] -(30) Sort [codegen id : 6] +(32) Sort [codegen id : 6] Input [2]: [item_sk#16, rnk#18] Arguments: [rnk#18 ASC NULLS FIRST], false, 0 -(31) SortMergeJoin [codegen id : 9] +(33) SortMergeJoin [codegen id : 9] Left keys [1]: [rnk#11] Right keys [1]: [rnk#18] Join type: Inner Join condition: None -(32) Project [codegen id : 9] +(34) Project [codegen id : 9] Output [3]: [item_sk#7, rnk#11, item_sk#16] Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] -(33) CometNativeScan: `spark_catalog`.`default`.`item` +(35) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_product_name#20] -Arguments: [i_item_sk#19, i_product_name#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(34) CometFilter +(36) CometFilter Input [2]: [i_item_sk#19, i_product_name#20] Condition : isnotnull(i_item_sk#19) -(35) ColumnarToRow [codegen id : 7] +(37) CometProject Input [2]: [i_item_sk#19, i_product_name#20] +Arguments: [i_item_sk#19, i_product_name#21], [i_item_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#20, 50, true, false, true) AS i_product_name#21] -(36) BroadcastExchange -Input [2]: [i_item_sk#19, i_product_name#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(38) CometColumnarToRow [codegen id : 7] +Input [2]: [i_item_sk#19, i_product_name#21] + +(39) BroadcastExchange +Input [2]: [i_item_sk#19, i_product_name#21] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 9] +(40) BroadcastHashJoin [codegen id : 9] Left keys [1]: [item_sk#7] Right keys [1]: [i_item_sk#19] Join type: Inner Join condition: None -(38) Project [codegen id : 9] -Output [3]: [rnk#11, item_sk#16, i_product_name#20] -Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#20] +(41) Project [codegen id : 9] +Output [3]: [rnk#11, item_sk#16, i_product_name#21] +Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#21] -(39) ReusedExchange [Reuses operator id: 36] -Output [2]: [i_item_sk#21, i_product_name#22] +(42) ReusedExchange [Reuses operator id: 39] +Output [2]: [i_item_sk#22, i_product_name#23] -(40) BroadcastHashJoin [codegen id : 9] +(43) BroadcastHashJoin [codegen id : 9] Left keys [1]: [item_sk#16] -Right keys [1]: [i_item_sk#21] +Right keys [1]: [i_item_sk#22] Join type: Inner Join condition: None -(41) Project [codegen id : 9] -Output [3]: [rnk#11, i_product_name#20 AS best_performing#23, i_product_name#22 AS worst_performing#24] -Input [5]: [rnk#11, item_sk#16, i_product_name#20, i_item_sk#21, i_product_name#22] +(44) Project [codegen id : 9] +Output [3]: [rnk#11, i_product_name#21 AS best_performing#24, i_product_name#23 AS worst_performing#25] +Input [5]: [rnk#11, item_sk#16, i_product_name#21, i_item_sk#22, i_product_name#23] -(42) TakeOrderedAndProject -Input [3]: [rnk#11, best_performing#23, worst_performing#24] -Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#23, worst_performing#24] +(45) TakeOrderedAndProject +Input [3]: [rnk#11, best_performing#24, worst_performing#25] +Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#24, worst_performing#25] ===== Subqueries ===== Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (49) -+- CometHashAggregate (48) - +- CometExchange (47) - +- CometHashAggregate (46) - +- CometProject (45) - +- CometFilter (44) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (43) - - -(43) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Arguments: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] - -(44) CometFilter -Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Condition : ((isnotnull(ss_store_sk#26) AND (ss_store_sk#26 = 4)) AND isnull(ss_addr_sk#25)) - -(45) CometProject -Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Arguments: [ss_store_sk#26, ss_net_profit#27], [ss_store_sk#26, ss_net_profit#27] - -(46) CometHashAggregate -Input [2]: [ss_store_sk#26, ss_net_profit#27] -Keys [1]: [ss_store_sk#26] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#27))] - -(47) CometExchange -Input [3]: [ss_store_sk#26, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(48) CometHashAggregate -Input [3]: [ss_store_sk#26, sum#29, count#30] -Keys [1]: [ss_store_sk#26] -Functions [1]: [avg(UnscaledValue(ss_net_profit#27))] - -(49) ColumnarToRow [codegen id : 1] -Input [1]: [rank_col#31] - -Subquery:2 Hosting operator id = 20 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +* CometColumnarToRow (52) ++- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometFilter (47) + +- CometNativeScan parquet spark_catalog.default.store_sales (46) + + +(46) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] +ReadSchema: struct + +(47) CometFilter +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_store_sk#27) AND (ss_store_sk#27 = 4)) AND isnull(ss_addr_sk#26)) + +(48) CometProject +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Arguments: [ss_store_sk#27, ss_net_profit#28], [ss_store_sk#27, ss_net_profit#28] + +(49) CometHashAggregate +Input [2]: [ss_store_sk#27, ss_net_profit#28] +Keys [1]: [ss_store_sk#27] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#28))] + +(50) CometExchange +Input [3]: [ss_store_sk#27, sum#30, count#31] +Arguments: hashpartitioning(ss_store_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(51) CometHashAggregate +Input [3]: [ss_store_sk#27, sum#30, count#31] +Keys [1]: [ss_store_sk#27] +Functions [1]: [avg(UnscaledValue(ss_net_profit#28))] + +(52) CometColumnarToRow [codegen id : 1] +Input [1]: [rank_col#32] + +Subquery:2 Hosting operator id = 21 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/extended.txt new file mode 100644 index 0000000000..9d58a4e060 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/extended.txt @@ -0,0 +1,64 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- SortMergeJoin + : : :- Sort + : : : +- Project + : : : +- Filter + : : : +- Window + : : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometColumnarExchange + : : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometFilter + : : : : +- Subquery + : : : : +- CometColumnarToRow + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- Sort + : : +- Project + : : +- Filter + : : +- Window + : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 32 out of 55 eligible operators (58%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/simplified.txt index 6b6c66a16e..a6db5c3397 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/simplified.txt @@ -15,31 +15,32 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Window [rank_col] WindowGroupLimit [rank_col] WholeStageCodegen (2) - Sort [rank_col] + CometColumnarToRow InputAdapter - Exchange #1 - WindowGroupLimit [rank_col] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [rank_col,ss_store_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometExchange [ss_store_sk] #3 - CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] - CometProject [ss_store_sk,ss_net_profit] - CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometExchange [ss_item_sk] #2 - CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometSort [item_sk,rank_col] + CometColumnarExchange #1 + WindowGroupLimit [rank_col] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [ss_store_sk,sum,count] [rank_col,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_store_sk] #3 + CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_item_sk] #2 + CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (6) Sort [rnk] @@ -49,24 +50,26 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Window [rank_col] WindowGroupLimit [rank_col] WholeStageCodegen (5) - Sort [rank_col] + CometColumnarToRow InputAdapter - Exchange #4 - WindowGroupLimit [rank_col] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - ReusedSubquery [rank_col] #1 - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - ReusedExchange [ss_item_sk,sum,count] #2 + CometSort [item_sk,rank_col] + CometColumnarExchange #4 + WindowGroupLimit [rank_col] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + ReusedSubquery [rank_col] #1 + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + ReusedExchange [ss_item_sk,sum,count] #2 InputAdapter BroadcastExchange #5 WholeStageCodegen (7) - ColumnarToRow + CometColumnarToRow InputAdapter - CometFilter [i_item_sk,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_product_name] + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/explain.txt index acee604c97..0856e37607 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/explain.txt @@ -1,53 +1,63 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`item` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +TakeOrderedAndProject (39) ++- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * Project (34) + +- * Filter (33) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (32) + :- * CometColumnarToRow (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- BroadcastExchange (31) + +- * CometColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) + +- CometNativeScan parquet spark_catalog.default.item (27) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Arguments: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#5)] +PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer` +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [c_customer_sk#6, c_current_addr_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [c_customer_sk#6, c_current_addr_sk#7] @@ -66,128 +76,151 @@ Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Condition : isnotnull(ca_address_sk#8) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#10, 10, true, false, true) AS ca_zip#11] + +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] -(11) CometBroadcastHashJoin +(12) CometBroadcastHashJoin Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight -(12) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11, d_year#12, d_qoy#13] - -(14) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) - -(15) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#5], [d_date_sk#11], Inner, BuildRight - -(18) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(20) CometFilter -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) - -(21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(22) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Right output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [ws_item_sk#2], [i_item_sk#14], Inner, BuildRight - -(23) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] -Arguments: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15], [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(24) ColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(25) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#17] - -(26) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) - -(27) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_id#17], [i_item_id#17] - -(28) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#17] - -(29) BroadcastExchange -Input [1]: [i_item_id#17] +(13) CometProject +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) + +(16) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] + +(18) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight + +(19) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] + +(20) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) + +(22) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] + +(24) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] + +(27) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] +ReadSchema: struct + +(28) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : i_item_sk#18 IN (2,3,5,7,11,13,17,19,23,29) + +(29) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_id#20], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] + +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [i_item_id#20] + +(31) BroadcastExchange +Input [1]: [i_item_id#20] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] -(30) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] +(32) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#20] Join type: ExistenceJoin(exists#1) Join condition: None -(31) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +(33) Filter [codegen id : 2] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) -(32) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] +(34) Project [codegen id : 2] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] -(33) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] +(35) HashAggregate [codegen id : 2] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] +Aggregate Attributes [1]: [sum#21] +Results [3]: [ca_zip#11, ca_city#9, sum#22] + +(36) CometColumnarExchange +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(34) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(37) CometColumnarToRow [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] -(35) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] +(38) HashAggregate [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#23] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#23,17,2) AS sum(ws_sales_price)#24] -(36) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] +(39) TakeOrderedAndProject +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/extended.txt new file mode 100644 index 0000000000..8fb8fa7188 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/extended.txt @@ -0,0 +1,41 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- Filter + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 29 out of 36 eligible operators (80%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/simplified.txt index 68d48dbfa2..fd8a442c1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/simplified.txt @@ -1,43 +1,46 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + CometColumnarToRow + InputAdapter + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 + CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/explain.txt index 1740385afd..944bb79698 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometProject (36) - +- CometBroadcastHashJoin (35) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) : :- CometHashAggregate (28) : : +- CometExchange (27) : : +- CometHashAggregate (26) @@ -17,39 +17,47 @@ : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : : : +- CometNativeScan parquet spark_catalog.default.store (9) : : : +- CometBroadcastExchange (18) : : : +- CometProject (17) : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- CometBroadcastExchange (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - +- ReusedExchange (34) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] @@ -72,9 +80,12 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_city#13] -Arguments: [s_store_sk#12, s_city#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#12, s_city#13] @@ -97,9 +108,12 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Arguments: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] @@ -122,9 +136,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(21) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ca_address_sk#17, ca_city#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct (22) CometFilter Input [2]: [ca_address_sk#17, ca_city#18] @@ -157,43 +174,50 @@ Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum# Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -(29) CometNativeScan: `spark_catalog`.`default`.`customer` +(29) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (30) CometFilter Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) -(31) CometBroadcastExchange +(31) CometProject Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [c_customer_sk#21, c_current_addr_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#26] + +(32) CometBroadcastExchange +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] -(32) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27] -Right output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +(33) CometBroadcastHashJoin +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29] +Right output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight -(33) CometProject -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Arguments: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24], [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +(34) CometProject +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29, c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] -(34) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#28, ca_city#29] +(35) ReusedExchange [Reuses operator id: 23] +Output [2]: [ca_address_sk#30, ca_city#31] -(35) CometBroadcastHashJoin -Left output [7]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Right output [2]: [ca_address_sk#28, ca_city#29] -Arguments: [c_current_addr_sk#22], [ca_address_sk#28], Inner, NOT (ca_city#29 = bought_city#25), BuildRight +(36) CometBroadcastHashJoin +Left output [7]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Right output [2]: [ca_address_sk#30, ca_city#31] +Arguments: [c_current_addr_sk#22], [ca_address_sk#30], Inner, NOT (ca_city#31 = bought_city#27), BuildRight -(36) CometProject -Input [9]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24, ca_address_sk#28, ca_city#29] -Arguments: [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27], [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] +(37) CometProject +Input [9]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26, ca_address_sk#30, ca_city#31] +Arguments: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] -(37) CometTakeOrderedAndProject -Input [7]: [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,ca_city#29 ASC NULLS FIRST,bought_city#25 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,ca_city#29,bought_city#25,ss_ticket_number#5,amt#26,profit#27]), [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27], 100, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, ca_city#29 ASC NULLS FIRST, bought_city#25 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] +(38) CometTakeOrderedAndProject +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#26 ASC NULLS FIRST,c_first_name#25 ASC NULLS FIRST,ca_city#31 ASC NULLS FIRST,bought_city#27 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#26,c_first_name#25,ca_city#31,bought_city#27,ss_ticket_number#5,amt#28,profit#29]), [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], 100, 0, [c_last_name#26 ASC NULLS FIRST, c_first_name#25 ASC NULLS FIRST, ca_city#31 ASC NULLS FIRST, bought_city#27 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] -(38) ColumnarToRow [codegen id : 1] -Input [7]: [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] +(39) CometColumnarToRow [codegen id : 1] +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/extended.txt new file mode 100644 index 0000000000..2cc4ab6ef2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 40 out of 40 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/simplified.txt index cf50411300..11f3dc34da 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] CometBroadcastHashJoin [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] CometProject [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,ss_addr_sk,ca_city,sum,sum,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] + CometHashAggregate [ss_addr_sk,sum,sum] [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,ca_city,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,ss_coupon_amt,ss_net_profit] + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] @@ -18,23 +18,24 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_city] CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/explain.txt index 02904e4ff1..b0e9f9ad7a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/explain.txt @@ -1,243 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight - -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(18) CometHashAggregate -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] - -(19) CometExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] - -(21) CometExchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] - -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#17], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Arguments: [avg(_w0#16) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] - -(27) Filter [codegen id : 7] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] -Condition : ((isnotnull(avg_monthly_sales#18) AND (avg_monthly_sales#18 > 0.000000)) AND CASE WHEN (avg_monthly_sales#18 > 0.000000) THEN ((abs((sum_sales#15 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] - -(29) ReusedExchange [Reuses operator id: 19] -Output [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] - -(30) CometHashAggregate -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -Keys [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) + +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight + +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] + +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] + +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] + +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] + +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] + +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) + +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] + +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(31) CometExchange -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: hashpartitioning(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15], [i_category#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(34) Window -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#19, i_brand#20, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#15 AS sum_sales#28, rn#27] -Input [8]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15, rn#27] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(36) BroadcastExchange -Input [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] -(39) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(40) CometSort -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15], [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(42) Window -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(43) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#15 AS sum_sales#36, rn#35] -Input [8]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15, rn#35] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] -(44) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(45) BroadcastExchange +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, (rn#35 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, sum_sales#28 AS psum#37, sum_sales#36 AS nsum#38] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28, i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(47) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] -(47) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] -Arguments: 100, [(sum_sales#15 - avg_monthly_sales#18) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] +(48) TakeOrderedAndProject +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/extended.txt new file mode 100644 index 0000000000..eca90ab6dc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/simplified.txt index 7594a9aaff..61ae95b11e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/explain.txt index ea30295898..01cb838bef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/explain.txt @@ -1,45 +1,53 @@ == Physical Plan == -* ColumnarToRow (28) -+- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometHashAggregate (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (13) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.store (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.customer_address (14) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometNativeScan parquet spark_catalog.default.date_dim (20) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] +ReadSchema: struct (2) CometFilter Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometNativeScan: `spark_catalog`.`default`.`store` +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [s_store_sk#8] @@ -58,91 +66,104 @@ Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(8) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) +Condition : (isnotnull(cd_demo_sk#9) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = College )))) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) AS cd_marital_status#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) AS cd_education_status#13] + +(11) CometBroadcastExchange +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] -(11) CometBroadcastHashJoin +(12) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight -(12) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +(13) CometProject +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(13) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] +(14) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct -(14) CometFilter -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) +(15) CometFilter +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (VA,CA,MS))) -(15) CometProject -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] +(16) CometProject +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) AS ca_state#17] -(16) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] +(17) CometBroadcastExchange +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] -(17) CometBroadcastHashJoin +(18) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#2], [ca_address_sk#12], Inner, ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight -(18) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] +(19) CometProject +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] +(20) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_year#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct -(20) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(21) CometFilter +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) -(21) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] +(22) CometProject +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] -(22) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] +(23) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight -(24) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] +(25) CometProject +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] Arguments: [ss_quantity#4], [ss_quantity#4] -(25) CometHashAggregate +(26) CometHashAggregate Input [1]: [ss_quantity#4] Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] -(26) CometExchange -Input [1]: [sum#17] +(27) CometExchange +Input [1]: [sum#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [1]: [sum#17] +(28) CometHashAggregate +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] -(28) ColumnarToRow [codegen id : 1] -Input [1]: [sum(ss_quantity)#18] +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/extended.txt new file mode 100644 index 0000000000..bdbe1b496a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/simplified.txt index aea9aa7cf6..5feebbec84 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [sum(ss_quantity),sum,sum(ss_quantity)] + CometHashAggregate [sum] [sum(ss_quantity),sum(ss_quantity)] CometExchange #1 - CometHashAggregate [sum,ss_quantity] + CometHashAggregate [ss_quantity] [sum] CometProject [ss_quantity] CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] CometProject [ss_quantity,ss_sold_date_sk] @@ -13,18 +13,19 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometProject [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/explain.txt index 33be1ecefe..1d6b3ef43f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/explain.txt @@ -1,56 +1,91 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- Union (43) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometExchange (19) - : +- CometHashAggregate (18) - : +- CometExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - :- * Project (34) - : +- * Filter (33) - : +- Window (32) - : +- * Sort (31) - : +- Window (30) - : +- * ColumnarToRow (29) - : +- CometSort (28) - : +- ReusedExchange (27) - +- * Project (42) - +- * Filter (41) - +- Window (40) - +- * Sort (39) - +- Window (38) - +- * ColumnarToRow (37) - +- CometSort (36) - +- ReusedExchange (35) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] +PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] @@ -64,9 +99,12 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(5) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct (6) CometFilter Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] @@ -85,9 +123,12 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(10) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_moy#14] @@ -132,7 +173,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(21) ColumnarToRow [codegen id : 1] +(21) CometColumnarToRow [codegen id : 1] Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window @@ -155,87 +196,230 @@ Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(27) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#27, return_ratio#28, currency_ratio#29] - -(28) CometSort -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [item#27, return_ratio#28, currency_ratio#29], [return_ratio#28 ASC NULLS FIRST] - -(29) ColumnarToRow [codegen id : 4] -Input [3]: [item#27, return_ratio#28, currency_ratio#29] - -(30) Window -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [rank(return_ratio#28) windowspecdefinition(return_ratio#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#30], [return_ratio#28 ASC NULLS FIRST] - -(31) Sort [codegen id : 5] -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [currency_ratio#29 ASC NULLS FIRST], false, 0 - -(32) Window -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [rank(currency_ratio#29) windowspecdefinition(currency_ratio#29 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#31], [currency_ratio#29 ASC NULLS FIRST] - -(33) Filter [codegen id : 6] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] -Condition : ((return_rank#30 <= 10) OR (currency_rank#31 <= 10)) - -(34) Project [codegen id : 6] -Output [5]: [catalog AS channel#32, item#27, return_ratio#28, return_rank#30, currency_rank#31] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] - -(35) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#33, return_ratio#34, currency_ratio#35] - -(36) CometSort -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [item#33, return_ratio#34, currency_ratio#35], [return_ratio#34 ASC NULLS FIRST] - -(37) ColumnarToRow [codegen id : 7] -Input [3]: [item#33, return_ratio#34, currency_ratio#35] - -(38) Window -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [rank(return_ratio#34) windowspecdefinition(return_ratio#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#36], [return_ratio#34 ASC NULLS FIRST] - -(39) Sort [codegen id : 8] -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [currency_ratio#35 ASC NULLS FIRST], false, 0 - -(40) Window -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [rank(currency_ratio#35) windowspecdefinition(currency_ratio#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#37], [currency_ratio#35 ASC NULLS FIRST] - -(41) Filter [codegen id : 9] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] -Condition : ((return_rank#36 <= 10) OR (currency_rank#37 <= 10)) - -(42) Project [codegen id : 9] -Output [5]: [store AS channel#38, item#33, return_ratio#34, return_rank#36, currency_rank#37] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] - -(43) Union - -(44) HashAggregate [codegen id : 10] +(27) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] +PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) + +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(31) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] +PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) + +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(54) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(45) Exchange +(75) CometColumnarExchange Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(46) HashAggregate [codegen id : 11] +(76) CometHashAggregate Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(47) TakeOrderedAndProject +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: 100, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/extended.txt new file mode 100644 index 0000000000..262571ed59 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/extended.txt @@ -0,0 +1,86 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- Filter + +- Window + +- Sort + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 63 out of 80 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/simplified.txt index baf07c5c9a..117b74a75b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/simplified.txt @@ -1,69 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (11) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/explain.txt index 61caf7d64e..03a193564c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/explain.txt @@ -1,61 +1,84 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Expand (48) - +- Union (47) - :- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - :- * HashAggregate (25) - : +- * ColumnarToRow (24) - : +- ReusedExchange (23) - +- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometExchange (44) - +- CometHashAggregate (43) - +- CometProject (42) - +- CometBroadcastHashJoin (41) - :- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometUnion (36) - : : :- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) - : : +- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (29) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (31) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometHashAggregate (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometExpand (66) + +- CometUnion (65) + :- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.store (14) + :- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometUnion (29) + : : : :- CometProject (25) + : : : : +- CometFilter (24) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (23) + : : : +- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (26) + : : +- ReusedExchange (30) + : +- CometBroadcastExchange (36) + : +- CometProject (35) + : +- CometFilter (34) + : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometUnion (52) + : : :- CometProject (44) + : : : +- CometFilter (43) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (42) + : : +- CometProject (51) + : : +- CometBroadcastHashJoin (50) + : : :- CometBroadcastExchange (46) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (45) + : : +- CometProject (49) + : : +- CometFilter (48) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (53) + +- CometBroadcastExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometNativeScan parquet spark_catalog.default.web_site (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] @@ -65,9 +88,13 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] @@ -81,9 +108,12 @@ Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, n Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21, d_date#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [d_date_sk#21, d_date#22] @@ -106,172 +136,274 @@ Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(14) CometNativeScan: `spark_catalog`.`default`.`store` +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (15) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(16) CometBroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] -(17) CometBroadcastHashJoin +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] +Right output [2]: [s_store_sk#23, s_store_id#25] Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(19) CometHashAggregate -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -(20) CometExchange -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) ColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Keys [1]: [s_store_id#24] +(22) CometHashAggregate +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#29, sum(UnscaledValue(return_amt#9))#30, sum(UnscaledValue(profit#8))#31, sum(UnscaledValue(net_loss#10))#32] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#29,17,2) AS sales#33, MakeDecimal(sum(UnscaledValue(return_amt#9))#30,17,2) AS returns#34, (MakeDecimal(sum(UnscaledValue(profit#8))#31,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#32,17,2)) AS profit#35, store channel AS channel#36, concat(store, s_store_id#24) AS id#37] - -(23) ReusedExchange [Reuses operator id: 20] -Output [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] - -(24) ColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] - -(25) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] -Keys [1]: [cp_catalog_page_id#38] -Functions [4]: [sum(UnscaledValue(sales_price#43)), sum(UnscaledValue(return_amt#44)), sum(UnscaledValue(profit#45)), sum(UnscaledValue(net_loss#46))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#43))#47, sum(UnscaledValue(return_amt#44))#48, sum(UnscaledValue(profit#45))#49, sum(UnscaledValue(net_loss#46))#50] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#43))#47,17,2) AS sales#51, MakeDecimal(sum(UnscaledValue(return_amt#44))#48,17,2) AS returns#52, (MakeDecimal(sum(UnscaledValue(profit#45))#49,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#46))#50,17,2)) AS profit#53, catalog channel AS channel#54, concat(catalog_page, cp_catalog_page_id#38) AS id#55] -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] +(23) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +PushedFilters: [IsNotNull(cs_catalog_page_sk)] +ReadSchema: struct + +(24) CometFilter +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : isnotnull(cs_catalog_page_sk#30) + +(25) CometProject +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39], [cs_catalog_page_sk#30 AS page_sk#34, cs_sold_date_sk#33 AS date_sk#35, cs_ext_sales_price#31 AS sales_price#36, cs_net_profit#32 AS profit#37, 0.00 AS return_amt#38, 0.00 AS net_loss#39] + +(26) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] +PushedFilters: [IsNotNull(cr_catalog_page_sk)] +ReadSchema: struct (27) CometFilter -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Condition : isnotnull(ws_web_site_sk#56) +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Condition : isnotnull(cr_catalog_page_sk#40) (28) CometProject -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65], [ws_web_site_sk#56 AS wsr_web_site_sk#60, ws_sold_date_sk#59 AS date_sk#61, ws_ext_sales_price#57 AS sales_price#62, ws_net_profit#58 AS profit#63, 0.00 AS return_amt#64, 0.00 AS net_loss#65] +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Arguments: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49], [cr_catalog_page_sk#40 AS page_sk#44, cr_returned_date_sk#43 AS date_sk#45, 0.00 AS sales_price#46, 0.00 AS profit#47, cr_return_amount#41 AS return_amt#48, cr_net_loss#42 AS net_loss#49] -(29) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] +(29) CometUnion +Child 0 Input [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Child 1 Input [6]: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49] -(30) CometBroadcastExchange -Input [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] +(30) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#50] -(31) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] +(31) CometBroadcastHashJoin +Left output [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [1]: [d_date_sk#50] +Arguments: [date_sk#35], [d_date_sk#50], Inner, BuildRight -(32) CometFilter -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_item_sk#71) AND isnotnull(ws_order_number#73)) AND isnotnull(ws_web_site_sk#72)) +(32) CometProject +Input [7]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39, d_date_sk#50] +Arguments: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39], [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] -(33) CometProject -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73], [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] +(33) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct -(34) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Right output [3]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wr_item_sk#66, wr_order_number#67], [ws_item_sk#71, ws_order_number#73], Inner, BuildLeft +(34) CometFilter +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Condition : isnotnull(cp_catalog_page_sk#51) (35) CometProject -Input [8]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70, ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80], [ws_web_site_sk#72 AS wsr_web_site_sk#75, wr_returned_date_sk#70 AS date_sk#76, 0.00 AS sales_price#77, 0.00 AS profit#78, wr_return_amt#68 AS return_amt#79, wr_net_loss#69 AS net_loss#80] - -(36) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Child 1 Input [6]: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80] - -(37) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#81] - -(38) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [1]: [d_date_sk#81] -Arguments: [date_sk#61], [d_date_sk#81], Inner, BuildRight - -(39) CometProject -Input [7]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65, d_date_sk#81] -Arguments: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65], [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] - -(40) ReusedExchange [Reuses operator id: 16] -Output [2]: [web_site_sk#82, web_site_id#83] - -(41) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [2]: [web_site_sk#82, web_site_id#83] -Arguments: [wsr_web_site_sk#60], [web_site_sk#82], Inner, BuildRight - -(42) CometProject -Input [7]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_sk#82, web_site_id#83] -Arguments: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83], [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] - -(43) CometHashAggregate -Input [5]: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] -Keys [1]: [web_site_id#83] -Functions [4]: [partial_sum(UnscaledValue(sales_price#62)), partial_sum(UnscaledValue(return_amt#64)), partial_sum(UnscaledValue(profit#63)), partial_sum(UnscaledValue(net_loss#65))] - -(44) CometExchange -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Arguments: hashpartitioning(web_site_id#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(45) ColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] - -(46) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Keys [1]: [web_site_id#83] -Functions [4]: [sum(UnscaledValue(sales_price#62)), sum(UnscaledValue(return_amt#64)), sum(UnscaledValue(profit#63)), sum(UnscaledValue(net_loss#65))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#62))#88, sum(UnscaledValue(return_amt#64))#89, sum(UnscaledValue(profit#63))#90, sum(UnscaledValue(net_loss#65))#91] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#62))#88,17,2) AS sales#92, MakeDecimal(sum(UnscaledValue(return_amt#64))#89,17,2) AS returns#93, (MakeDecimal(sum(UnscaledValue(profit#63))#90,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#65))#91,17,2)) AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#83) AS id#96] - -(47) Union - -(48) Expand [codegen id : 4] -Input [5]: [sales#33, returns#34, profit#35, channel#36, id#37] -Arguments: [[sales#33, returns#34, profit#35, channel#36, id#37, 0], [sales#33, returns#34, profit#35, channel#36, null, 1], [sales#33, returns#34, profit#35, null, null, 3]], [sales#33, returns#34, profit#35, channel#97, id#98, spark_grouping_id#99] - -(49) HashAggregate [codegen id : 4] -Input [6]: [sales#33, returns#34, profit#35, channel#97, id#98, spark_grouping_id#99] -Keys [3]: [channel#97, id#98, spark_grouping_id#99] -Functions [3]: [partial_sum(sales#33), partial_sum(returns#34), partial_sum(profit#35)] -Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] - -(50) Exchange -Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(51) HashAggregate [codegen id : 5] -Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [3]: [channel#97, id#98, spark_grouping_id#99] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [5]: [channel#97, id#98, sum(sales#33)#112 AS sales#115, sum(returns#34)#113 AS returns#116, sum(profit#35)#114 AS profit#117] - -(52) TakeOrderedAndProject -Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] -Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53], [cp_catalog_page_sk#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#52, 16, true, false, true) AS cp_catalog_page_id#53] + +(36) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53] + +(37) CometBroadcastHashJoin +Left output [5]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [page_sk#34], [cp_catalog_page_sk#51], Inner, BuildRight + +(38) CometProject +Input [7]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53], [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] + +(39) CometHashAggregate +Input [5]: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [partial_sum(UnscaledValue(sales_price#36)), partial_sum(UnscaledValue(return_amt#38)), partial_sum(UnscaledValue(profit#37)), partial_sum(UnscaledValue(net_loss#39))] + +(40) CometExchange +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(41) CometHashAggregate +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [sum(UnscaledValue(sales_price#36)), sum(UnscaledValue(return_amt#38)), sum(UnscaledValue(profit#37)), sum(UnscaledValue(net_loss#39))] + +(42) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] +PushedFilters: [IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(43) CometFilter +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_web_site_sk#58) + +(44) CometProject +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Arguments: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67], [ws_web_site_sk#58 AS wsr_web_site_sk#62, ws_sold_date_sk#61 AS date_sk#63, ws_ext_sales_price#59 AS sales_price#64, ws_net_profit#60 AS profit#65, 0.00 AS return_amt#66, 0.00 AS net_loss#67] + +(45) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#72)] +ReadSchema: struct + +(46) CometBroadcastExchange +Input [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Arguments: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] + +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(48) CometFilter +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Condition : ((isnotnull(ws_item_sk#73) AND isnotnull(ws_order_number#75)) AND isnotnull(ws_web_site_sk#74)) + +(49) CometProject +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Arguments: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75], [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] + +(50) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Right output [3]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wr_item_sk#68, wr_order_number#69], [ws_item_sk#73, ws_order_number#75], Inner, BuildLeft + +(51) CometProject +Input [8]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72, ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82], [ws_web_site_sk#74 AS wsr_web_site_sk#77, wr_returned_date_sk#72 AS date_sk#78, 0.00 AS sales_price#79, 0.00 AS profit#80, wr_return_amt#70 AS return_amt#81, wr_net_loss#71 AS net_loss#82] + +(52) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Child 1 Input [6]: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82] + +(53) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#83] + +(54) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [1]: [d_date_sk#83] +Arguments: [date_sk#63], [d_date_sk#83], Inner, BuildRight + +(55) CometProject +Input [7]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67, d_date_sk#83] +Arguments: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67], [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] + +(56) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#84, web_site_id#85] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(57) CometFilter +Input [2]: [web_site_sk#84, web_site_id#85] +Condition : isnotnull(web_site_sk#84) + +(58) CometProject +Input [2]: [web_site_sk#84, web_site_id#85] +Arguments: [web_site_sk#84, web_site_id#86], [web_site_sk#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#85, 16, true, false, true) AS web_site_id#86] + +(59) CometBroadcastExchange +Input [2]: [web_site_sk#84, web_site_id#86] +Arguments: [web_site_sk#84, web_site_id#86] + +(60) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [2]: [web_site_sk#84, web_site_id#86] +Arguments: [wsr_web_site_sk#62], [web_site_sk#84], Inner, BuildRight + +(61) CometProject +Input [7]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_sk#84, web_site_id#86] +Arguments: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86], [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] + +(62) CometHashAggregate +Input [5]: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] +Keys [1]: [web_site_id#86] +Functions [4]: [partial_sum(UnscaledValue(sales_price#64)), partial_sum(UnscaledValue(return_amt#66)), partial_sum(UnscaledValue(profit#65)), partial_sum(UnscaledValue(net_loss#67))] + +(63) CometExchange +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Arguments: hashpartitioning(web_site_id#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(64) CometHashAggregate +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Keys [1]: [web_site_id#86] +Functions [4]: [sum(UnscaledValue(sales_price#64)), sum(UnscaledValue(return_amt#66)), sum(UnscaledValue(profit#65)), sum(UnscaledValue(net_loss#67))] + +(65) CometUnion +Child 0 Input [5]: [sales#91, returns#92, profit#93, channel#94, id#95] +Child 1 Input [5]: [sales#96, returns#97, profit#98, channel#99, id#100] +Child 2 Input [5]: [sales#101, returns#102, profit#103, channel#104, id#105] + +(66) CometExpand +Input [5]: [sales#91, returns#92, profit#93, channel#94, id#95] +Arguments: [[sales#91, returns#92, profit#93, channel#94, id#95, 0], [sales#91, returns#92, profit#93, channel#94, null, 1], [sales#91, returns#92, profit#93, null, null, 3]], [sales#91, returns#92, profit#93, channel#106, id#107, spark_grouping_id#108] + +(67) CometHashAggregate +Input [6]: [sales#91, returns#92, profit#93, channel#106, id#107, spark_grouping_id#108] +Keys [3]: [channel#106, id#107, spark_grouping_id#108] +Functions [3]: [partial_sum(sales#91), partial_sum(returns#92), partial_sum(profit#93)] + +(68) CometExchange +Input [9]: [channel#106, id#107, spark_grouping_id#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Arguments: hashpartitioning(channel#106, id#107, spark_grouping_id#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(69) CometHashAggregate +Input [9]: [channel#106, id#107, spark_grouping_id#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Keys [3]: [channel#106, id#107, spark_grouping_id#108] +Functions [3]: [sum(sales#91), sum(returns#92), sum(profit#93)] + +(70) CometTakeOrderedAndProject +Input [5]: [channel#106, id#107, sales#115, returns#116, profit#117] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#106 ASC NULLS FIRST,id#107 ASC NULLS FIRST], output=[channel#106,id#107,sales#115,returns#116,profit#117]), [channel#106, id#107, sales#115, returns#116, profit#117], 100, 0, [channel#106 ASC NULLS FIRST, id#107 ASC NULLS FIRST], [channel#106, id#107, sales#115, returns#116, profit#117] + +(71) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#106, id#107, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/extended.txt new file mode 100644 index 0000000000..ebcc9b408b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/extended.txt @@ -0,0 +1,79 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_page + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 76 out of 76 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/simplified.txt index 3f9946093f..09633b6ce5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/simplified.txt @@ -1,62 +1,73 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [cp_catalog_page_id,sum,sum,sum,sum] #2 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #5 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #6 - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [web_site_sk,web_site_id] #4 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id,spark_grouping_id] #1 + CometHashAggregate [sales,returns,profit] [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometExpand [channel,id] [sales,returns,profit,channel,id,spark_grouping_id] + CometUnion [sales,returns,profit,channel,id] + CometHashAggregate [sum,sum,sum,sum] [sales,returns,profit,channel,id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),s_store_id] + CometExchange [s_store_id] #2 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometHashAggregate [sum,sum,sum,sum] [sales,returns,profit,channel,id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),cp_catalog_page_id] + CometExchange [cp_catalog_page_id] #5 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometHashAggregate [sum,sum,sum,sum] [sales,returns,profit,channel,id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),web_site_id] + CometExchange [web_site_id] #7 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/explain.txt index 49e3721e6c..1fe7a0246b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/explain.txt @@ -1,45 +1,54 @@ == Physical Plan == -* ColumnarToRow (28) -+- CometTakeOrderedAndProject (27) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.store (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#9)] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -58,91 +67,104 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_numb Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -(8) CometNativeScan: `spark_catalog`.`default`.`store` +(8) CometNativeScan parquet spark_catalog.default.store Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (9) CometFilter Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Condition : isnotnull(s_store_sk#10) -(10) CometBroadcastExchange +(10) CometProject Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_street_type#15, 15, true, false, true) AS s_street_type#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_suite_number#16, 10, true, false, true) AS s_suite_number#22, s_city#17, s_county#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#19, 2, true, false, true) AS s_state#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#20, 10, true, false, true) AS s_zip#24] + +(11) CometBroadcastExchange +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(11) CometBroadcastHashJoin +(12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(12) CometProject -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(14) CometFilter -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(16) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#21] -Arguments: [ss_sold_date_sk#5], [d_date_sk#21], Inner, BuildRight - -(17) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22, d_year#23, d_moy#24] - -(19) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) - -(20) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(21) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] - -(22) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#9], [d_date_sk#22], Inner, BuildRight - -(23) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(24) CometHashAggregate -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(13) CometProject +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [1]: [d_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) + +(16) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(17) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight + +(18) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) + +(21) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] + +(23) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight + +(24) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(25) CometHashAggregate +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometExchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#25, sum#26, sum#27, sum#28, sum#29] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometExchange +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) CometHashAggregate -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#25, sum#26, sum#27, sum#28, sum#29] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(27) CometHashAggregate +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(27) CometTakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#15 ASC NULLS FIRST,s_suite_number#16 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#19 ASC NULLS FIRST,s_zip#20 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#15,s_suite_number#16,s_city#17,s_county#18,s_state#19,s_zip#20,30 days #30,31 - 60 days #31,61 - 90 days #32,91 - 120 days #33,>120 days #34]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34], 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] +(28) CometTakeOrderedAndProject +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] -(28) ColumnarToRow [codegen id : 1] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] +(29) CometColumnarToRow [codegen id : 1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/extended.txt new file mode 100644 index 0000000000..a188fea58c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/simplified.txt index b6e1187f76..e53cdee055 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum,sum,sum] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum,sr_returned_date_sk,ss_sold_date_sk] + CometHashAggregate [sr_returned_date_sk,ss_sold_date_sk] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] @@ -14,17 +14,18 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastExchange [d_date_sk] #4 CometFilter [d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/explain.txt index 6e0895aa3e..f9c8d14c62 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/explain.txt @@ -1,44 +1,65 @@ == Physical Plan == -TakeOrderedAndProject (27) -+- * Filter (26) - +- Window (25) - +- * Sort (24) - +- Exchange (23) - +- * Project (22) - +- * SortMergeJoin FullOuter (21) - :- * Sort (18) - : +- Exchange (17) - : +- * Project (16) - : +- Window (15) - : +- * ColumnarToRow (14) - : +- CometSort (13) - : +- CometExchange (12) - : +- CometHashAggregate (11) - : +- CometExchange (10) - : +- CometHashAggregate (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- * Sort (20) - +- ReusedExchange (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +TakeOrderedAndProject (41) ++- * Filter (40) + +- Window (39) + +- * CometColumnarToRow (38) + +- CometSort (37) + +- CometExchange (36) + +- CometProject (35) + +- CometSortMergeJoin (34) + :- CometSort (18) + : +- CometColumnarExchange (17) + : +- * Project (16) + : +- Window (15) + : +- * CometColumnarToRow (14) + : +- CometSort (13) + : +- CometExchange (12) + : +- CometHashAggregate (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- CometSort (33) + +- CometColumnarExchange (32) + +- * Project (31) + +- Window (30) + +- * CometColumnarToRow (29) + +- CometSort (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometFilter (20) + : +- CometNativeScan parquet spark_catalog.default.store_sales (19) + +- ReusedExchange (21) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] @@ -83,7 +104,7 @@ Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] Arguments: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(14) ColumnarToRow [codegen id : 1] +(14) CometColumnarToRow [codegen id : 1] Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] (15) Window @@ -94,48 +115,108 @@ Arguments: [sum(_w0#9) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIR Output [3]: [item_sk#8, d_date#5, cume_sales#10] Input [5]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1, cume_sales#10] -(17) Exchange +(17) CometColumnarExchange Input [3]: [item_sk#8, d_date#5, cume_sales#10] -Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(18) Sort [codegen id : 3] +(18) CometSort Input [3]: [item_sk#8, d_date#5, cume_sales#10] -Arguments: [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(19) ReusedExchange [Reuses operator id: 17] -Output [3]: [item_sk#11, d_date#12, cume_sales#13] - -(20) Sort [codegen id : 6] -Input [3]: [item_sk#11, d_date#12, cume_sales#13] -Arguments: [item_sk#11 ASC NULLS FIRST, d_date#12 ASC NULLS FIRST], false, 0 - -(21) SortMergeJoin [codegen id : 7] -Left keys [2]: [item_sk#8, d_date#5] -Right keys [2]: [item_sk#11, d_date#12] -Join type: FullOuter -Join condition: None - -(22) Project [codegen id : 7] -Output [4]: [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#11 END AS item_sk#14, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#12 END AS d_date#15, cume_sales#10 AS web_sales#16, cume_sales#13 AS store_sales#17] -Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#11, d_date#12, cume_sales#13] - -(23) Exchange -Input [4]: [item_sk#14, d_date#15, web_sales#16, store_sales#17] -Arguments: hashpartitioning(item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 8] -Input [4]: [item_sk#14, d_date#15, web_sales#16, store_sales#17] -Arguments: [item_sk#14 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST], false, 0 - -(25) Window -Input [4]: [item_sk#14, d_date#15, web_sales#16, store_sales#17] -Arguments: [max(web_sales#16) windowspecdefinition(item_sk#14, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#18, max(store_sales#17) windowspecdefinition(item_sk#14, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#19], [item_sk#14], [d_date#15 ASC NULLS FIRST] - -(26) Filter [codegen id : 9] -Input [6]: [item_sk#14, d_date#15, web_sales#16, store_sales#17, web_cumulative#18, store_cumulative#19] -Condition : ((isnotnull(web_cumulative#18) AND isnotnull(store_cumulative#19)) AND (web_cumulative#18 > store_cumulative#19)) - -(27) TakeOrderedAndProject -Input [6]: [item_sk#14, d_date#15, web_sales#16, store_sales#17, web_cumulative#18, store_cumulative#19] -Arguments: 100, [item_sk#14 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST], [item_sk#14, d_date#15, web_sales#16, store_sales#17, web_cumulative#18, store_cumulative#19] +Arguments: [item_sk#8, d_date#5, cume_sales#10], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(19) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(20) CometFilter +Input [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#11) + +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#14, d_date#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_date#15] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(23) CometProject +Input [5]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_date#15] +Arguments: [ss_item_sk#11, ss_sales_price#12, d_date#15], [ss_item_sk#11, ss_sales_price#12, d_date#15] + +(24) CometHashAggregate +Input [3]: [ss_item_sk#11, ss_sales_price#12, d_date#15] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] + +(25) CometExchange +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Arguments: hashpartitioning(ss_item_sk#11, d_date#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometHashAggregate +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] + +(27) CometExchange +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: hashpartitioning(ss_item_sk#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(28) CometSort +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11], [ss_item_sk#11 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] + +(29) CometColumnarToRow [codegen id : 3] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] + +(30) Window +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [sum(_w0#18) windowspecdefinition(ss_item_sk#11, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#19], [ss_item_sk#11], [d_date#15 ASC NULLS FIRST] + +(31) Project [codegen id : 4] +Output [3]: [item_sk#17, d_date#15, cume_sales#19] +Input [5]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11, cume_sales#19] + +(32) CometColumnarExchange +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: hashpartitioning(item_sk#17, d_date#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(33) CometSort +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#17, d_date#15, cume_sales#19], [item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] + +(34) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#10] +Right output [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#8, d_date#5], [item_sk#17, d_date#15], FullOuter + +(35) CometProject +Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#17 END AS item_sk#20, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#15 END AS d_date#21, cume_sales#10 AS web_sales#22, cume_sales#19 AS store_sales#23] + +(36) CometExchange +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: hashpartitioning(item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(37) CometSort +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST] + +(38) CometColumnarToRow [codegen id : 5] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] + +(39) Window +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [max(web_sales#22) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#24, max(store_sales#23) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#25], [item_sk#20], [d_date#21 ASC NULLS FIRST] + +(40) Filter [codegen id : 6] +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Condition : ((isnotnull(web_cumulative#24) AND isnotnull(store_cumulative#25)) AND (web_cumulative#24 > store_cumulative#25)) + +(41) TakeOrderedAndProject +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Arguments: 100, [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST], [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/extended.txt new file mode 100644 index 0000000000..08195fdffe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/extended.txt @@ -0,0 +1,46 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometColumnarExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 34 out of 41 eligible operators (82%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/simplified.txt index 9e95125e39..300f5044c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/simplified.txt @@ -1,42 +1,53 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (9) + WholeStageCodegen (6) Filter [web_cumulative,store_cumulative] InputAdapter Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (8) - Sort [item_sk,d_date] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (7) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (3) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (2) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,_w0,ws_item_sk] - CometExchange [ws_item_sk] #3 - CometHashAggregate [item_sk,d_date,_w0,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometExchange [ws_item_sk,d_date] #4 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,cume_sales] #2 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (2) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometExchange [ws_item_sk] #3 + CometHashAggregate [sum] [item_sk,d_date,_w0,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #6 + WholeStageCodegen (4) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometExchange [ss_item_sk] #7 + CometHashAggregate [sum] [item_sk,d_date,_w0,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #8 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/explain.txt index b9de030234..dec8071d59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,19 +10,22 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] @@ -63,39 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnul (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] (15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [4]: [d_year#2, brand_id#12, brand#13, ext_price#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,ext_price#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[d_year#2,brand_id#12,brand#13,ext_price#14]), [d_year#2, brand_id#12, brand#13, ext_price#14], 100, [d_year#2 ASC NULLS FIRST, ext_price#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [d_year#2, brand_id#12, brand#13, ext_price#14] +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,ext_price#15]), [d_year#2, brand_id#13, brand#14, ext_price#15], 100, 0, [d_year#2 ASC NULLS FIRST, ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, ext_price#15] -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, brand_id#12, brand#13, ext_price#14] +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/simplified.txt index 24f033cc73..cee223d027 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,ext_price] - CometHashAggregate [d_year,brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [d_year,brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [d_year,i_brand,i_brand_id] #1 - CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] CometProject [d_year,ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/explain.txt index 827fbeb9d3..e09d5b7575 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/explain.txt @@ -3,7 +3,7 @@ TakeOrderedAndProject (29) +- * Project (28) +- * Filter (27) +- Window (26) - +- * ColumnarToRow (25) + +- * CometColumnarToRow (25) +- CometSort (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -17,34 +17,41 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) + +- CometNativeScan parquet spark_catalog.default.store (15) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,reference ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) (3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -63,9 +70,12 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] @@ -88,9 +98,12 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [1]: [s_store_sk#17] @@ -131,7 +144,7 @@ Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativ Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] Arguments: [i_manufact_id#5, sum_sales#19, _w0#20], [i_manufact_id#5 ASC NULLS FIRST] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] (26) Window diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/extended.txt new file mode 100644 index 0000000000..44fd48691b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/extended.txt @@ -0,0 +1,31 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/simplified.txt index c4574bc31b..7e4a2be5ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/simplified.txt @@ -5,13 +5,13 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter Window [_w0,i_manufact_id] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_manufact_id,sum_sales,_w0] CometExchange [i_manufact_id] #1 - CometHashAggregate [i_manufact_id,sum_sales,_w0,d_qoy,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [d_qoy,sum] [i_manufact_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_manufact_id,d_qoy] #2 - CometHashAggregate [i_manufact_id,d_qoy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_manufact_id,d_qoy,sum] CometProject [i_manufact_id,ss_sales_price,d_qoy] CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] @@ -20,14 +20,14 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manufact_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_qoy] #4 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/explain.txt index 40376430a5..f153c6f850 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/explain.txt @@ -1,18 +1,18 @@ == Physical Plan == -* ColumnarToRow (56) -+- CometTakeOrderedAndProject (55) - +- CometHashAggregate (54) - +- CometExchange (53) - +- CometHashAggregate (52) - +- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (37) - : : +- CometBroadcastHashJoin (36) +* CometColumnarToRow (58) ++- CometTakeOrderedAndProject (57) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometHashAggregate (53) + +- CometExchange (52) + +- CometHashAggregate (51) + +- CometProject (50) + +- CometBroadcastHashJoin (49) + :- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (38) + : : +- CometBroadcastHashJoin (37) : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometHashAggregate (27) @@ -27,39 +27,45 @@ : : : : : : :- CometUnion (7) : : : : : : : :- CometProject (3) : : : : : : : : +- CometFilter (2) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : : : : +- CometProject (6) : : : : : : : +- CometFilter (5) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (4) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (4) : : : : : : +- CometBroadcastExchange (11) : : : : : : +- CometProject (10) : : : : : : +- CometFilter (9) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (8) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (8) : : : : : +- CometBroadcastExchange (17) : : : : : +- CometProject (16) : : : : : +- CometFilter (15) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) : : : : +- CometBroadcastExchange (22) : : : : +- CometFilter (21) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (20) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (30) : : : +- CometFilter (29) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (28) - : : +- CometBroadcastExchange (35) - : : +- CometFilter (34) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (33) - : +- CometBroadcastExchange (40) - : +- CometFilter (39) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (38) - +- CometBroadcastExchange (46) - +- CometProject (45) - +- CometFilter (44) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (43) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (28) + : : +- CometBroadcastExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (33) + : +- CometBroadcastExchange (42) + : +- CometProject (41) + : +- CometFilter (40) + : +- CometNativeScan parquet spark_catalog.default.store (39) + +- CometBroadcastExchange (48) + +- CometProject (47) + +- CometFilter (46) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] @@ -69,9 +75,13 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] -(4) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(4) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] -Arguments: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] @@ -85,13 +95,16 @@ Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] -(8) CometNativeScan: `spark_catalog`.`default`.`item` +(8) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#13, i_class#14, i_category#15] -Arguments: [i_item_sk#13, i_class#14, i_category#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [i_item_sk#13, i_class#14, i_category#15] -Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (i_category#15 = Women )) AND (i_class#14 = maternity )) AND isnotnull(i_item_sk#13)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#14, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#13)) (10) CometProject Input [3]: [i_item_sk#13, i_class#14, i_category#15] @@ -110,9 +123,12 @@ Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct (15) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] @@ -135,9 +151,12 @@ Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] Arguments: [customer_sk#5], [customer_sk#5] -(20) CometNativeScan: `spark_catalog`.`default`.`customer` +(20) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: [c_customer_sk#19, c_current_addr_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (21) CometFilter Input [2]: [c_customer_sk#19, c_current_addr_sk#20] @@ -170,9 +189,13 @@ Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] -(28) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(28) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (29) CometFilter Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] @@ -191,186 +214,209 @@ Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -(33) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(33) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [ca_address_sk#24, ca_county#25, ca_state#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] +ReadSchema: struct (34) CometFilter Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true))) -(35) CometBroadcastExchange +(35) CometProject Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [ca_address_sk#24, ca_county#25, ca_state#26] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true) AS ca_state#27] + +(36) CometBroadcastExchange +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] -(36) CometBroadcastHashJoin +(37) CometBroadcastHashJoin Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight -(37) CometProject -Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] +(38) CometProject +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] -(38) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_county#27, s_state#28] -Arguments: [s_county#27, s_state#28] +(39) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_county#28, s_state#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county)] +ReadSchema: struct -(39) CometFilter -Input [2]: [s_county#27, s_state#28] -Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) +(40) CometFilter +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true))) -(40) CometBroadcastExchange -Input [2]: [s_county#27, s_state#28] -Arguments: [s_county#27, s_state#28] +(41) CometProject +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true) AS s_state#30] -(41) CometBroadcastHashJoin -Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] -Right output [2]: [s_county#27, s_state#28] -Arguments: [ca_county#25, ca_state#26], [s_county#27, s_state#28], Inner, BuildRight +(42) CometBroadcastExchange +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] -(42) CometProject -Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26, s_county#27, s_state#28] +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -(43) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#29, d_month_seq#30] -Arguments: [d_date_sk#29, d_month_seq#30] +(45) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_month_seq#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(44) CometFilter -Input [2]: [d_date_sk#29, d_month_seq#30] -Condition : (((isnotnull(d_month_seq#30) AND (d_month_seq#30 >= Subquery scalar-subquery#31, [id=#32])) AND (d_month_seq#30 <= Subquery scalar-subquery#33, [id=#34])) AND isnotnull(d_date_sk#29)) +(46) CometFilter +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= Subquery scalar-subquery#33, [id=#34])) AND (d_month_seq#32 <= Subquery scalar-subquery#35, [id=#36])) AND isnotnull(d_date_sk#31)) -(45) CometProject -Input [2]: [d_date_sk#29, d_month_seq#30] -Arguments: [d_date_sk#29], [d_date_sk#29] +(47) CometProject +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] -(46) CometBroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: [d_date_sk#29] +(48) CometBroadcastExchange +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] -(47) CometBroadcastHashJoin +(49) CometBroadcastHashJoin Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -Right output [1]: [d_date_sk#29] -Arguments: [ss_sold_date_sk#23], [d_date_sk#29], Inner, BuildRight +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight -(48) CometProject -Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#29] +(50) CometProject +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] -(49) CometHashAggregate +(51) CometHashAggregate Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] Keys [1]: [c_customer_sk#19] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] -(50) CometExchange -Input [2]: [c_customer_sk#19, sum#35] +(52) CometExchange +Input [2]: [c_customer_sk#19, sum#37] Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(51) CometHashAggregate -Input [2]: [c_customer_sk#19, sum#35] +(53) CometHashAggregate +Input [2]: [c_customer_sk#19, sum#37] Keys [1]: [c_customer_sk#19] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] -(52) CometHashAggregate -Input [1]: [segment#36] -Keys [1]: [segment#36] +(54) CometHashAggregate +Input [1]: [segment#38] +Keys [1]: [segment#38] Functions [1]: [partial_count(1)] -(53) CometExchange -Input [2]: [segment#36, count#37] -Arguments: hashpartitioning(segment#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(55) CometExchange +Input [2]: [segment#38, count#39] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(54) CometHashAggregate -Input [2]: [segment#36, count#37] -Keys [1]: [segment#36] +(56) CometHashAggregate +Input [2]: [segment#38, count#39] +Keys [1]: [segment#38] Functions [1]: [count(1)] -(55) CometTakeOrderedAndProject -Input [3]: [segment#36, num_customers#38, segment_base#39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#36 ASC NULLS FIRST,num_customers#38 ASC NULLS FIRST], output=[segment#36,num_customers#38,segment_base#39]), [segment#36, num_customers#38, segment_base#39], 100, [segment#36 ASC NULLS FIRST, num_customers#38 ASC NULLS FIRST], [segment#36, num_customers#38, segment_base#39] +(57) CometTakeOrderedAndProject +Input [3]: [segment#38, num_customers#40, segment_base#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#38 ASC NULLS FIRST,num_customers#40 ASC NULLS FIRST], output=[segment#38,num_customers#40,segment_base#41]), [segment#38, num_customers#40, segment_base#41], 100, 0, [segment#38 ASC NULLS FIRST, num_customers#40 ASC NULLS FIRST], [segment#38, num_customers#40, segment_base#41] -(56) ColumnarToRow [codegen id : 1] -Input [3]: [segment#36, num_customers#38, segment_base#39] +(58) CometColumnarToRow [codegen id : 1] +Input [3]: [segment#38, num_customers#40, segment_base#41] ===== Subqueries ===== -Subquery:1 Hosting operator id = 44 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* ColumnarToRow (63) -+- CometHashAggregate (62) - +- CometExchange (61) - +- CometHashAggregate (60) - +- CometProject (59) - +- CometFilter (58) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (57) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#33, [id=#34] +* CometColumnarToRow (65) ++- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometNativeScan parquet spark_catalog.default.date_dim (59) -(57) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#40, d_year#41, d_moy#42] -Arguments: [d_month_seq#40, d_year#41, d_moy#42] +(59) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#42, d_year#43, d_moy#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] +ReadSchema: struct -(58) CometFilter -Input [3]: [d_month_seq#40, d_year#41, d_moy#42] -Condition : (((isnotnull(d_year#41) AND isnotnull(d_moy#42)) AND (d_year#41 = 1998)) AND (d_moy#42 = 12)) +(60) CometFilter +Input [3]: [d_month_seq#42, d_year#43, d_moy#44] +Condition : (((isnotnull(d_year#43) AND isnotnull(d_moy#44)) AND (d_year#43 = 1998)) AND (d_moy#44 = 12)) -(59) CometProject -Input [3]: [d_month_seq#40, d_year#41, d_moy#42] -Arguments: [(d_month_seq + 1)#43], [(d_month_seq#40 + 1) AS (d_month_seq + 1)#43] +(61) CometProject +Input [3]: [d_month_seq#42, d_year#43, d_moy#44] +Arguments: [(d_month_seq + 1)#45], [(d_month_seq#42 + 1) AS (d_month_seq + 1)#45] -(60) CometHashAggregate -Input [1]: [(d_month_seq + 1)#43] -Keys [1]: [(d_month_seq + 1)#43] +(62) CometHashAggregate +Input [1]: [(d_month_seq + 1)#45] +Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(61) CometExchange -Input [1]: [(d_month_seq + 1)#43] -Arguments: hashpartitioning((d_month_seq + 1)#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(63) CometExchange +Input [1]: [(d_month_seq + 1)#45] +Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(62) CometHashAggregate -Input [1]: [(d_month_seq + 1)#43] -Keys [1]: [(d_month_seq + 1)#43] +(64) CometHashAggregate +Input [1]: [(d_month_seq + 1)#45] +Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(63) ColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 1)#43] +(65) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#45] -Subquery:2 Hosting operator id = 44 Hosting Expression = Subquery scalar-subquery#33, [id=#34] -* ColumnarToRow (70) -+- CometHashAggregate (69) - +- CometExchange (68) - +- CometHashAggregate (67) - +- CometProject (66) - +- CometFilter (65) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (64) +Subquery:2 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#35, [id=#36] +* CometColumnarToRow (72) ++- CometHashAggregate (71) + +- CometExchange (70) + +- CometHashAggregate (69) + +- CometProject (68) + +- CometFilter (67) + +- CometNativeScan parquet spark_catalog.default.date_dim (66) -(64) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#44, d_year#45, d_moy#46] -Arguments: [d_month_seq#44, d_year#45, d_moy#46] +(66) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#46, d_year#47, d_moy#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] +ReadSchema: struct -(65) CometFilter -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Condition : (((isnotnull(d_year#45) AND isnotnull(d_moy#46)) AND (d_year#45 = 1998)) AND (d_moy#46 = 12)) +(67) CometFilter +Input [3]: [d_month_seq#46, d_year#47, d_moy#48] +Condition : (((isnotnull(d_year#47) AND isnotnull(d_moy#48)) AND (d_year#47 = 1998)) AND (d_moy#48 = 12)) -(66) CometProject -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Arguments: [(d_month_seq + 3)#47], [(d_month_seq#44 + 3) AS (d_month_seq + 3)#47] +(68) CometProject +Input [3]: [d_month_seq#46, d_year#47, d_moy#48] +Arguments: [(d_month_seq + 3)#49], [(d_month_seq#46 + 3) AS (d_month_seq + 3)#49] -(67) CometHashAggregate -Input [1]: [(d_month_seq + 3)#47] -Keys [1]: [(d_month_seq + 3)#47] +(69) CometHashAggregate +Input [1]: [(d_month_seq + 3)#49] +Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(68) CometExchange -Input [1]: [(d_month_seq + 3)#47] -Arguments: hashpartitioning((d_month_seq + 3)#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(70) CometExchange +Input [1]: [(d_month_seq + 3)#49] +Arguments: hashpartitioning((d_month_seq + 3)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(69) CometHashAggregate -Input [1]: [(d_month_seq + 3)#47] -Keys [1]: [(d_month_seq + 3)#47] +(71) CometHashAggregate +Input [1]: [(d_month_seq + 3)#49] +Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(70) ColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 3)#47] +(72) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/extended.txt new file mode 100644 index 0000000000..f51f7f1fe6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/extended.txt @@ -0,0 +1,76 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometUnion + : : : : : : : :- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometProject + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : :- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- Subquery + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 69 out of 71 eligible operators (97%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/simplified.txt index 4956f2825d..65164c2ea4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/simplified.txt @@ -1,13 +1,13 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [segment,num_customers,segment_base] - CometHashAggregate [segment,num_customers,segment_base,count,count(1)] + CometHashAggregate [count] [segment,num_customers,segment_base,count(1)] CometExchange [segment] #1 CometHashAggregate [segment,count] - CometHashAggregate [segment,c_customer_sk,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [c_customer_sk,sum] [segment,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [c_customer_sk] #2 - CometHashAggregate [c_customer_sk,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [c_customer_sk,sum] CometProject [c_customer_sk,ss_ext_sales_price] CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] @@ -28,51 +28,53 @@ WholeStageCodegen (1) CometUnion [sold_date_sk,customer_sk,item_sk] CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] CometBroadcastExchange [i_item_sk] #4 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 - CometFilter [ca_address_sk,ca_county,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state] + CometProject [ca_state] [ca_address_sk,ca_county,ca_state] + CometFilter [ca_address_sk,ca_county,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] CometBroadcastExchange [s_county,s_state] #9 - CometFilter [s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_county,s_state] + CometProject [s_state] [s_county,s_state] + CometFilter [s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_county,s_state] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 1)] CometExchange [(d_month_seq + 1)] #11 CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #2 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 3)] CometExchange [(d_month_seq + 3)] #12 CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/explain.txt index b06aa5ec9b..db0a08d421 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,19 +10,22 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] @@ -63,39 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnu (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] (15) CometHashAggregate -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] +Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Keys [2]: [i_brand#9, i_brand_id#8] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [3]: [brand_id#12, brand#13, ext_price#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[brand_id#12,brand#13,ext_price#14]), [brand_id#12, brand#13, ext_price#14], 100, [ext_price#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [brand_id#12, brand#13, ext_price#14] +Input [3]: [brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[brand_id#13,brand#14,ext_price#15]), [brand_id#13, brand#14, ext_price#15], 100, 0, [ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [brand_id#13, brand#14, ext_price#15] -(19) ColumnarToRow [codegen id : 1] -Input [3]: [brand_id#12, brand#13, ext_price#14] +(19) CometColumnarToRow [codegen id : 1] +Input [3]: [brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/simplified.txt index 737a74576d..a2adacc531 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [brand_id,brand,ext_price] - CometHashAggregate [brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_brand,i_brand_id] #1 - CometHashAggregate [i_brand,i_brand_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,sum] CometProject [ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] CometProject [ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/explain.txt index fbf528bb90..d6176f5bd9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/explain.txt @@ -1,66 +1,86 @@ == Physical Plan == -* ColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometHashAggregate (47) - +- CometExchange (46) - +- CometHashAggregate (45) - +- CometUnion (44) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- CometHashAggregate (43) - +- ReusedExchange (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] @@ -83,9 +103,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] @@ -108,151 +131,218 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometNativeScan: `spark_catalog`.`default`.`item` +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12, i_color#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct (18) CometFilter Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) +Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#13, 20, true, false, true) IN (slate ,blanched ,burnished ) (19) CometProject Input [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12], [i_item_id#12] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(22) CometBroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] -(23) CometBroadcastHashJoin +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] +Right output [2]: [i_item_sk#10, i_item_id#15] Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometExchange -Input [2]: [i_item_id#11, sum#14] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [2]: [i_item_id#11, sum#14] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(29) CometFilter -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) - -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct -(31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) -(32) CometProject -Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#20] +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight -(34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] -Right output [1]: [ca_address_sk#20] -Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] -(35) CometProject -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] -Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] -(36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#21, i_item_id#22] +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight -(37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] -Right output [2]: [i_item_sk#21, i_item_id#22] -Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] -(38) CometProject -Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_item_id#22] -Arguments: [cs_ext_sales_price#17, i_item_id#22], [cs_ext_sales_price#17, i_item_id#22] +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] -(39) CometHashAggregate -Input [2]: [cs_ext_sales_price#17, i_item_id#22] -Keys [1]: [i_item_id#22] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight -(40) CometExchange -Input [2]: [i_item_id#22, sum#23] -Arguments: hashpartitioning(i_item_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] -(41) CometHashAggregate -Input [2]: [i_item_id#22, sum#23] -Keys [1]: [i_item_id#22] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] -(42) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_item_id#24, sum#25] +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(43) CometHashAggregate +(42) CometHashAggregate Input [2]: [i_item_id#24, sum#25] Keys [1]: [i_item_id#24] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] - -(44) CometUnion -Child 0 Input [2]: [i_item_id#11, total_sales#27] -Child 1 Input [2]: [i_item_id#22, total_sales#28] -Child 2 Input [2]: [i_item_id#24, total_sales#29] - -(45) CometHashAggregate -Input [2]: [i_item_id#11, total_sales#27] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#27)] - -(46) CometExchange -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(47) CometHashAggregate -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#27)] - -(48) CometTakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#32 ASC NULLS FIRST], output=[i_item_id#11,total_sales#32]), [i_item_id#11, total_sales#32], 100, [total_sales#32 ASC NULLS FIRST], [i_item_id#11, total_sales#32] - -(49) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#11, total_sales#32] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] + +(43) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/extended.txt new file mode 100644 index 0000000000..7d3fb6fd9a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/extended.txt @@ -0,0 +1,92 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 89 out of 89 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/simplified.txt index 56464c9b57..7cfa14025c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,isEmpty,total_sales] + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] CometUnion [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id] #2 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_ext_sales_price] @@ -16,26 +16,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_color] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_color] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_color] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_color] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #7 - CometHashAggregate [i_item_id,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_ext_sales_price] @@ -43,9 +44,21 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [i_item_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/explain.txt index 1e3ab6de8f..1bc39b2ef4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/explain.txt @@ -1,243 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(18) CometHashAggregate -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] - -(19) CometExchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] - -(21) CometExchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] - -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#16], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Arguments: [avg(_w0#15) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#17], [i_category#3, i_brand#2, cc_name#12, d_year#9] - -(27) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] -Condition : ((isnotnull(avg_monthly_sales#17) AND (avg_monthly_sales#17 > 0.000000)) AND CASE WHEN (avg_monthly_sales#17 > 0.000000) THEN ((abs((sum_sales#14 - avg_monthly_sales#17)) / avg_monthly_sales#17) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] - -(29) ReusedExchange [Reuses operator id: 19] -Output [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] - -(30) CometHashAggregate -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] -Keys [5]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight + +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] + +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] + +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] + +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] + +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] + +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) + +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] + +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] + +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(31) CometExchange -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: hashpartitioning(i_category#18, i_brand#19, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14], [i_category#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(34) Window -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#18, i_brand#19, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#18, i_brand#19, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#14 AS sum_sales#26, rn#25] -Input [7]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14, rn#25] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(36) BroadcastExchange -Input [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#18, i_brand#19, cc_name#20, (rn#25 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] -(39) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(40) CometSort -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14], [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(42) Window -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(43) Window +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#14 AS sum_sales#33, rn#32] -Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14, rn#32] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] -(44) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(45) BroadcastExchange +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#32 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, sum_sales#26 AS psum#34, sum_sales#33 AS nsum#35] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26, i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(47) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] -(47) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] -Arguments: 100, [(sum_sales#14 - avg_monthly_sales#17) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] +(48) TakeOrderedAndProject +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/extended.txt new file mode 100644 index 0000000000..357307b654 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.call_center + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/simplified.txt index 9666c6c7ae..1a3c0e7f18 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,cc_name] #1 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,cc_name] #7 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/explain.txt index f9908c19a1..cda35f1b45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/explain.txt @@ -1,198 +1,337 @@ == Physical Plan == -* ColumnarToRow (33) -+- CometTakeOrderedAndProject (32) - +- CometProject (31) - +- CometBroadcastHashJoin (30) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometFilter (22) - : : +- CometHashAggregate (21) - : : +- CometExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (55) ++- CometTakeOrderedAndProject (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometFilter (23) + : : +- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (13) - : : +- CometProject (12) - : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometHashAggregate (24) - : +- ReusedExchange (23) - +- ReusedExchange (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) + : : :- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (11) + : +- CometBroadcastExchange (36) + : +- CometFilter (35) + : +- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (51) + +- CometFilter (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometProject (46) + +- CometBroadcastHashJoin (45) + :- CometProject (43) + : +- CometBroadcastHashJoin (42) + : :- CometFilter (40) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (41) + +- ReusedExchange (44) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] + +(7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] +Right output [2]: [i_item_sk#4, i_item_id#6] Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6, d_date#7] - -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8, d_week_seq#9] - -(11) CometFilter -Input [2]: [d_date#8, d_week_seq#9] -Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery scalar-subquery#10, [id=#11])) - -(12) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] - -(13) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] - -(14) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight - -(15) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(17) CometBroadcastHashJoin -Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#3], [d_date_sk#6], Inner, BuildRight - -(18) CometProject -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [ss_ext_sales_price#2, i_item_id#5], [ss_ext_sales_price#2, i_item_id#5] - -(19) CometHashAggregate -Input [2]: [ss_ext_sales_price#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(8) CometProject +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) + +(11) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq)] +ReadSchema: struct + +(12) CometFilter +Input [2]: [d_date#9, d_week_seq#10] +Condition : (isnotnull(d_week_seq#10) AND (d_week_seq#10 = Subquery scalar-subquery#11, [id=#12])) + +(13) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] + +(14) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] + +(15) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight + +(16) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] + +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(20) CometExchange -Input [2]: [i_item_id#5, sum#12] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) CometHashAggregate -Input [2]: [i_item_id#5, sum#12] -Keys [1]: [i_item_id#5] +(22) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(22) CometFilter -Input [2]: [item_id#13, ss_item_rev#14] -Condition : isnotnull(ss_item_rev#14) +(23) CometFilter +Input [2]: [item_id#14, ss_item_rev#15] +Condition : isnotnull(ss_item_rev#15) -(23) ReusedExchange [Reuses operator id: 20] -Output [2]: [i_item_id#15, sum#16] - -(24) CometHashAggregate -Input [2]: [i_item_id#15, sum#16] -Keys [1]: [i_item_id#15] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] +(24) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (25) CometFilter -Input [2]: [item_id#18, cs_item_rev#19] -Condition : isnotnull(cs_item_rev#19) +Input [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#16) -(26) CometBroadcastExchange -Input [2]: [item_id#18, cs_item_rev#19] -Arguments: [item_id#18, cs_item_rev#19] +(26) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#19, i_item_id#20] (27) CometBroadcastHashJoin -Left output [2]: [item_id#13, ss_item_rev#14] -Right output [2]: [item_id#18, cs_item_rev#19] -Arguments: [item_id#13], [item_id#18], Inner, ((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * cs_item_rev#19)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * cs_item_rev#19))) AND (cast(cs_item_rev#19 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(cs_item_rev#19 as decimal(20,3)) <= (1.1 * ss_item_rev#14))), BuildRight +Left output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [2]: [i_item_sk#19, i_item_id#20] +Arguments: [cs_item_sk#16], [i_item_sk#19], Inner, BuildRight (28) CometProject -Input [4]: [item_id#13, ss_item_rev#14, item_id#18, cs_item_rev#19] -Arguments: [item_id#13, ss_item_rev#14, cs_item_rev#19], [item_id#13, ss_item_rev#14, cs_item_rev#19] +Input [5]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_sk#19, i_item_id#20] +Arguments: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20], [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] -(29) ReusedExchange [Reuses operator id: 26] -Output [2]: [item_id#20, ws_item_rev#21] +(29) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#21] (30) CometBroadcastHashJoin -Left output [3]: [item_id#13, ss_item_rev#14, cs_item_rev#19] -Right output [2]: [item_id#20, ws_item_rev#21] -Arguments: [item_id#13], [item_id#20], Inner, ((((((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * ws_item_rev#21)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * ws_item_rev#21))) AND (cast(cs_item_rev#19 as decimal(19,3)) >= (0.9 * ws_item_rev#21))) AND (cast(cs_item_rev#19 as decimal(20,3)) <= (1.1 * ws_item_rev#21))) AND (cast(ws_item_rev#21 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(ws_item_rev#21 as decimal(20,3)) <= (1.1 * ss_item_rev#14))) AND (cast(ws_item_rev#21 as decimal(19,3)) >= (0.9 * cs_item_rev#19))) AND (cast(ws_item_rev#21 as decimal(20,3)) <= (1.1 * cs_item_rev#19))), BuildRight +Left output [3]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#18], [d_date_sk#21], Inner, BuildRight (31) CometProject -Input [5]: [item_id#13, ss_item_rev#14, cs_item_rev#19, item_id#20, ws_item_rev#21] -Arguments: [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25], [item_id#13, ss_item_rev#14, (((ss_item_rev#14 / ((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21)) / 3) * 100) AS ss_dev#22, cs_item_rev#19, (((cs_item_rev#19 / ((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21)) / 3) * 100) AS cs_dev#23, ws_item_rev#21, (((ws_item_rev#21 / ((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21)) / 3) * 100) AS ws_dev#24, (((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21) / 3) AS average#25] +Input [4]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20, d_date_sk#21] +Arguments: [cs_ext_sales_price#17, i_item_id#20], [cs_ext_sales_price#17, i_item_id#20] -(32) CometTakeOrderedAndProject -Input [8]: [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#13 ASC NULLS FIRST,ss_item_rev#14 ASC NULLS FIRST], output=[item_id#13,ss_item_rev#14,ss_dev#22,cs_item_rev#19,cs_dev#23,ws_item_rev#21,ws_dev#24,average#25]), [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25], 100, [item_id#13 ASC NULLS FIRST, ss_item_rev#14 ASC NULLS FIRST], [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25] +(32) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_item_id#20] +Keys [1]: [i_item_id#20] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] -(33) ColumnarToRow [codegen id : 1] -Input [8]: [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25] +(33) CometExchange +Input [2]: [i_item_id#20, sum#22] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(34) CometHashAggregate +Input [2]: [i_item_id#20, sum#22] +Keys [1]: [i_item_id#20] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] + +(35) CometFilter +Input [2]: [item_id#23, cs_item_rev#24] +Condition : isnotnull(cs_item_rev#24) + +(36) CometBroadcastExchange +Input [2]: [item_id#23, cs_item_rev#24] +Arguments: [item_id#23, cs_item_rev#24] + +(37) CometBroadcastHashJoin +Left output [2]: [item_id#14, ss_item_rev#15] +Right output [2]: [item_id#23, cs_item_rev#24] +Arguments: [item_id#14], [item_id#23], Inner, ((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * cs_item_rev#24)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * cs_item_rev#24))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ss_item_rev#15))), BuildRight + +(38) CometProject +Input [4]: [item_id#14, ss_item_rev#15, item_id#23, cs_item_rev#24] +Arguments: [item_id#14, ss_item_rev#15, cs_item_rev#24], [item_id#14, ss_item_rev#15, cs_item_rev#24] + +(39) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : isnotnull(ws_item_sk#25) + +(41) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#28, i_item_id#29] + +(42) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Right output [2]: [i_item_sk#28, i_item_id#29] +Arguments: [ws_item_sk#25], [i_item_sk#28], Inner, BuildRight + +(43) CometProject +Input [5]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29], [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29] + +(44) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#30] + +(45) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#27], [d_date_sk#30], Inner, BuildRight + +(46) CometProject +Input [4]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [ws_ext_sales_price#26, i_item_id#29], [ws_ext_sales_price#26, i_item_id#29] + +(47) CometHashAggregate +Input [2]: [ws_ext_sales_price#26, i_item_id#29] +Keys [1]: [i_item_id#29] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] + +(48) CometExchange +Input [2]: [i_item_id#29, sum#31] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(49) CometHashAggregate +Input [2]: [i_item_id#29, sum#31] +Keys [1]: [i_item_id#29] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] + +(50) CometFilter +Input [2]: [item_id#32, ws_item_rev#33] +Condition : isnotnull(ws_item_rev#33) + +(51) CometBroadcastExchange +Input [2]: [item_id#32, ws_item_rev#33] +Arguments: [item_id#32, ws_item_rev#33] + +(52) CometBroadcastHashJoin +Left output [3]: [item_id#14, ss_item_rev#15, cs_item_rev#24] +Right output [2]: [item_id#32, ws_item_rev#33] +Arguments: [item_id#14], [item_id#32], Inner, ((((((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * ws_item_rev#33)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ws_item_rev#33))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * ss_item_rev#15))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * cs_item_rev#24))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * cs_item_rev#24))), BuildRight + +(53) CometProject +Input [5]: [item_id#14, ss_item_rev#15, cs_item_rev#24, item_id#32, ws_item_rev#33] +Arguments: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], [item_id#14, ss_item_rev#15, (((ss_item_rev#15 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS ss_dev#34, cs_item_rev#24, (((cs_item_rev#24 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS cs_dev#35, ws_item_rev#33, (((ws_item_rev#33 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS ws_dev#36, (((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33) / 3) AS average#37] + +(54) CometTakeOrderedAndProject +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#14 ASC NULLS FIRST,ss_item_rev#15 ASC NULLS FIRST], output=[item_id#14,ss_item_rev#15,ss_dev#34,cs_item_rev#24,cs_dev#35,ws_item_rev#33,ws_dev#36,average#37]), [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], 100, 0, [item_id#14 ASC NULLS FIRST, ss_item_rev#15 ASC NULLS FIRST], [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] + +(55) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] ===== Subqueries ===== -Subquery:1 Hosting operator id = 11 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (37) -+- CometProject (36) - +- CometFilter (35) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) +Subquery:1 Hosting operator id = 12 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (59) ++- CometProject (58) + +- CometFilter (57) + +- CometNativeScan parquet spark_catalog.default.date_dim (56) -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#26, d_week_seq#27] -Arguments: [d_date#26, d_week_seq#27] +(56) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#38, d_week_seq#39] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] +ReadSchema: struct -(35) CometFilter -Input [2]: [d_date#26, d_week_seq#27] -Condition : (isnotnull(d_date#26) AND (d_date#26 = 2000-01-03)) +(57) CometFilter +Input [2]: [d_date#38, d_week_seq#39] +Condition : (isnotnull(d_date#38) AND (d_date#38 = 2000-01-03)) -(36) CometProject -Input [2]: [d_date#26, d_week_seq#27] -Arguments: [d_week_seq#27], [d_week_seq#27] +(58) CometProject +Input [2]: [d_date#38, d_week_seq#39] +Arguments: [d_week_seq#39], [d_week_seq#39] -(37) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#27] +(59) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/extended.txt new file mode 100644 index 0000000000..4ed6f209f2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/extended.txt @@ -0,0 +1,94 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : +- Subquery + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 85 out of 88 eligible operators (96%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/simplified.txt index c0a83eeddd..cda22df7d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] CometProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] @@ -7,36 +7,58 @@ WholeStageCodegen (1) CometProject [item_id,ss_item_rev,cs_item_rev] CometBroadcastHashJoin [item_id,ss_item_rev,item_id,cs_item_rev] CometFilter [item_id,ss_item_rev] - CometHashAggregate [item_id,ss_item_rev,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [item_id,ss_item_rev,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #4 CometProject [d_date] CometFilter [d_date,d_week_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [item_id,cs_item_rev] #5 CometFilter [item_id,cs_item_rev] - CometHashAggregate [item_id,cs_item_rev,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] - ReusedExchange [i_item_id,sum] #1 - ReusedExchange [item_id,ws_item_rev] #5 + CometHashAggregate [sum] [item_id,cs_item_rev,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id] #6 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [item_id,ws_item_rev] #7 + CometFilter [item_id,ws_item_rev] + CometHashAggregate [sum] [item_id,ws_item_rev,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/explain.txt index 0235ad1c25..1bb4b7dc8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/explain.txt @@ -1,205 +1,239 @@ == Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (21) - : +- CometBroadcastHashJoin (20) - : :- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometHashAggregate (10) - : : : +- CometExchange (9) - : : : +- CometHashAggregate (8) - : : : +- CometProject (7) - : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (42) ++- CometTakeOrderedAndProject (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (13) - : : +- CometFilter (12) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (11) - : +- CometBroadcastExchange (19) - : +- CometProject (18) - : +- CometFilter (17) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - +- CometBroadcastExchange (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometHashAggregate (23) - : : +- ReusedExchange (22) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (24) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.store (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.date_dim (18) + +- CometBroadcastExchange (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometHashAggregate (25) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.store (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) -(5) CometBroadcastExchange +(5) CometProject Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7], [d_date_sk#4, d_week_seq#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#6, 9, true, false, true) AS d_day_name#7] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7] + +(7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(7) CometProject -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(8) CometProject +Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] -(8) CometHashAggregate -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(9) CometHashAggregate +Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(9) CometExchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] +(10) CometExchange +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(10) CometHashAggregate -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] +(11) CometHashAggregate +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] - -(11) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [s_store_sk#14, s_store_id#15, s_store_name#16] - -(12) CometFilter -Input [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Condition : (isnotnull(s_store_sk#14) AND isnotnull(s_store_id#15)) - -(13) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [s_store_sk#14, s_store_id#15, s_store_name#16] - -(14) CometBroadcastHashJoin -Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23] -Right output [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [ss_store_sk#1], [s_store_sk#14], Inner, BuildRight - -(15) CometProject -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16], [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16] - -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#24, d_week_seq#25] -Arguments: [d_month_seq#24, d_week_seq#25] - -(17) CometFilter -Input [2]: [d_month_seq#24, d_week_seq#25] -Condition : (((isnotnull(d_month_seq#24) AND (d_month_seq#24 >= 1212)) AND (d_month_seq#24 <= 1223)) AND isnotnull(d_week_seq#25)) - -(18) CometProject -Input [2]: [d_month_seq#24, d_week_seq#25] -Arguments: [d_week_seq#25], [d_week_seq#25] - -(19) CometBroadcastExchange -Input [1]: [d_week_seq#25] -Arguments: [d_week_seq#25] - -(20) CometBroadcastHashJoin -Left output [10]: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16] -Right output [1]: [d_week_seq#25] -Arguments: [d_week_seq#5], [d_week_seq#25], Inner, BuildRight - -(21) CometProject -Input [11]: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16, d_week_seq#25] -Arguments: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35], [s_store_name#16 AS s_store_name1#26, d_week_seq#5 AS d_week_seq1#27, s_store_id#15 AS s_store_id1#28, sun_sales#17 AS sun_sales1#29, mon_sales#18 AS mon_sales1#30, tue_sales#19 AS tue_sales1#31, wed_sales#20 AS wed_sales1#32, thu_sales#21 AS thu_sales1#33, fri_sales#22 AS fri_sales1#34, sat_sales#23 AS sat_sales1#35] - -(22) ReusedExchange [Reuses operator id: 9] -Output [9]: [d_week_seq#36, ss_store_sk#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] - -(23) CometHashAggregate -Input [9]: [d_week_seq#36, ss_store_sk#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] -Keys [2]: [d_week_seq#36, ss_store_sk#37] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#45 = Sunday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Monday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Tuesday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Wednesday) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Thursday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Friday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Saturday ) THEN ss_sales_price#46 END))] - -(24) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#47, s_store_id#48] -Arguments: [s_store_sk#47, s_store_id#48] - -(25) CometFilter -Input [2]: [s_store_sk#47, s_store_id#48] -Condition : (isnotnull(s_store_sk#47) AND isnotnull(s_store_id#48)) - -(26) CometBroadcastExchange -Input [2]: [s_store_sk#47, s_store_id#48] -Arguments: [s_store_sk#47, s_store_id#48] - -(27) CometBroadcastHashJoin -Left output [9]: [d_week_seq#36, ss_store_sk#37, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55] -Right output [2]: [s_store_sk#47, s_store_id#48] -Arguments: [ss_store_sk#37], [s_store_sk#47], Inner, BuildRight +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] + +(12) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(13) CometFilter +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Condition : (isnotnull(s_store_sk#15) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true))) + +(14) CometProject +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#18, s_store_name#17] + +(15) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17] + +(16) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25] +Right output [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight + +(17) CometProject +Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] + +(18) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#26, d_week_seq#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [d_month_seq#26, d_week_seq#27] +Condition : (((isnotnull(d_month_seq#26) AND (d_month_seq#26 >= 1212)) AND (d_month_seq#26 <= 1223)) AND isnotnull(d_week_seq#27)) + +(20) CometProject +Input [2]: [d_month_seq#26, d_week_seq#27] +Arguments: [d_week_seq#27], [d_week_seq#27] + +(21) CometBroadcastExchange +Input [1]: [d_week_seq#27] +Arguments: [d_week_seq#27] + +(22) CometBroadcastHashJoin +Left output [10]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] +Right output [1]: [d_week_seq#27] +Arguments: [d_week_seq#5], [d_week_seq#27], Inner, BuildRight + +(23) CometProject +Input [11]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17, d_week_seq#27] +Arguments: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37], [s_store_name#17 AS s_store_name1#28, d_week_seq#5 AS d_week_seq1#29, s_store_id#18 AS s_store_id1#30, sun_sales#19 AS sun_sales1#31, mon_sales#20 AS mon_sales1#32, tue_sales#21 AS tue_sales1#33, wed_sales#22 AS wed_sales1#34, thu_sales#23 AS thu_sales1#35, fri_sales#24 AS fri_sales1#36, sat_sales#25 AS sat_sales1#37] + +(24) ReusedExchange [Reuses operator id: 10] +Output [9]: [d_week_seq#38, ss_store_sk#39, sum#40, sum#41, sum#42, sum#43, sum#44, sum#45, sum#46] + +(25) CometHashAggregate +Input [9]: [d_week_seq#38, ss_store_sk#39, sum#40, sum#41, sum#42, sum#43, sum#44, sum#45, sum#46] +Keys [2]: [d_week_seq#38, ss_store_sk#39] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#47 END))] + +(26) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#48, s_store_id#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [s_store_sk#48, s_store_id#49] +Condition : (isnotnull(s_store_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#49, 16, true, false, true))) (28) CometProject -Input [11]: [d_week_seq#36, ss_store_sk#37, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_sk#47, s_store_id#48] -Arguments: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48], [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48] +Input [2]: [s_store_sk#48, s_store_id#49] +Arguments: [s_store_sk#48, s_store_id#50], [s_store_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#49, 16, true, false, true) AS s_store_id#50] -(29) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#56, d_week_seq#57] -Arguments: [d_month_seq#56, d_week_seq#57] +(29) CometBroadcastExchange +Input [2]: [s_store_sk#48, s_store_id#50] +Arguments: [s_store_sk#48, s_store_id#50] -(30) CometFilter -Input [2]: [d_month_seq#56, d_week_seq#57] -Condition : (((isnotnull(d_month_seq#56) AND (d_month_seq#56 >= 1224)) AND (d_month_seq#56 <= 1235)) AND isnotnull(d_week_seq#57)) +(30) CometBroadcastHashJoin +Left output [9]: [d_week_seq#38, ss_store_sk#39, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57] +Right output [2]: [s_store_sk#48, s_store_id#50] +Arguments: [ss_store_sk#39], [s_store_sk#48], Inner, BuildRight (31) CometProject -Input [2]: [d_month_seq#56, d_week_seq#57] -Arguments: [d_week_seq#57], [d_week_seq#57] +Input [11]: [d_week_seq#38, ss_store_sk#39, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_sk#48, s_store_id#50] +Arguments: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50], [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50] -(32) CometBroadcastExchange -Input [1]: [d_week_seq#57] -Arguments: [d_week_seq#57] +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#58, d_week_seq#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] +ReadSchema: struct -(33) CometBroadcastHashJoin -Left output [9]: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48] -Right output [1]: [d_week_seq#57] -Arguments: [d_week_seq#36], [d_week_seq#57], Inner, BuildRight +(33) CometFilter +Input [2]: [d_month_seq#58, d_week_seq#59] +Condition : (((isnotnull(d_month_seq#58) AND (d_month_seq#58 >= 1224)) AND (d_month_seq#58 <= 1235)) AND isnotnull(d_week_seq#59)) (34) CometProject -Input [10]: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48, d_week_seq#57] -Arguments: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66], [d_week_seq#36 AS d_week_seq2#58, s_store_id#48 AS s_store_id2#59, sun_sales#49 AS sun_sales2#60, mon_sales#50 AS mon_sales2#61, tue_sales#51 AS tue_sales2#62, wed_sales#52 AS wed_sales2#63, thu_sales#53 AS thu_sales2#64, fri_sales#54 AS fri_sales2#65, sat_sales#55 AS sat_sales2#66] +Input [2]: [d_month_seq#58, d_week_seq#59] +Arguments: [d_week_seq#59], [d_week_seq#59] (35) CometBroadcastExchange -Input [9]: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] +Input [1]: [d_week_seq#59] +Arguments: [d_week_seq#59] (36) CometBroadcastHashJoin -Left output [10]: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35] -Right output [9]: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [s_store_id1#28, d_week_seq1#27], [s_store_id2#59, (d_week_seq2#58 - 52)], Inner, BuildRight +Left output [9]: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50] +Right output [1]: [d_week_seq#59] +Arguments: [d_week_seq#38], [d_week_seq#59], Inner, BuildRight (37) CometProject -Input [19]: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35, d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73], [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1#29 / sun_sales2#60) AS (sun_sales1 / sun_sales2)#67, (mon_sales1#30 / mon_sales2#61) AS (mon_sales1 / mon_sales2)#68, (tue_sales1#31 / tue_sales2#62) AS (tue_sales1 / tue_sales2)#69, (wed_sales1#32 / wed_sales2#63) AS (wed_sales1 / wed_sales2)#70, (thu_sales1#33 / thu_sales2#64) AS (thu_sales1 / thu_sales2)#71, (fri_sales1#34 / fri_sales2#65) AS (fri_sales1 / fri_sales2)#72, (sat_sales1#35 / sat_sales2#66) AS (sat_sales1 / sat_sales2)#73] +Input [10]: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50, d_week_seq#59] +Arguments: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68], [d_week_seq#38 AS d_week_seq2#60, s_store_id#50 AS s_store_id2#61, sun_sales#51 AS sun_sales2#62, mon_sales#52 AS mon_sales2#63, tue_sales#53 AS tue_sales2#64, wed_sales#54 AS wed_sales2#65, thu_sales#55 AS thu_sales2#66, fri_sales#56 AS fri_sales2#67, sat_sales#57 AS sat_sales2#68] + +(38) CometBroadcastExchange +Input [9]: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] +Arguments: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] + +(39) CometBroadcastHashJoin +Left output [10]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37] +Right output [9]: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] +Arguments: [s_store_id1#30, d_week_seq1#29], [s_store_id2#61, (d_week_seq2#60 - 52)], Inner, BuildRight + +(40) CometProject +Input [19]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37, d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] +Arguments: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1#31 / sun_sales2#62) AS (sun_sales1 / sun_sales2)#69, (mon_sales1#32 / mon_sales2#63) AS (mon_sales1 / mon_sales2)#70, (tue_sales1#33 / tue_sales2#64) AS (tue_sales1 / tue_sales2)#71, (wed_sales1#34 / wed_sales2#65) AS (wed_sales1 / wed_sales2)#72, (thu_sales1#35 / thu_sales2#66) AS (thu_sales1 / thu_sales2)#73, (fri_sales1#36 / fri_sales2#67) AS (fri_sales1 / fri_sales2)#74, (sat_sales1#37 / sat_sales2#68) AS (sat_sales1 / sat_sales2)#75] -(38) CometTakeOrderedAndProject -Input [10]: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name1#26 ASC NULLS FIRST,s_store_id1#28 ASC NULLS FIRST,d_week_seq1#27 ASC NULLS FIRST], output=[s_store_name1#26,s_store_id1#28,d_week_seq1#27,(sun_sales1 / sun_sales2)#67,(mon_sales1 / mon_sales2)#68,(tue_sales1 / tue_sales2)#69,(wed_sales1 / wed_sales2)#70,(thu_sales1 / thu_sales2)#71,(fri_sales1 / fri_sales2)#72,(sat_sales1 / sat_sales2)#73]), [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73], 100, [s_store_name1#26 ASC NULLS FIRST, s_store_id1#28 ASC NULLS FIRST, d_week_seq1#27 ASC NULLS FIRST], [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] +(41) CometTakeOrderedAndProject +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name1#28 ASC NULLS FIRST,s_store_id1#30 ASC NULLS FIRST,d_week_seq1#29 ASC NULLS FIRST], output=[s_store_name1#28,s_store_id1#30,d_week_seq1#29,(sun_sales1 / sun_sales2)#69,(mon_sales1 / mon_sales2)#70,(tue_sales1 / tue_sales2)#71,(wed_sales1 / wed_sales2)#72,(thu_sales1 / thu_sales2)#73,(fri_sales1 / fri_sales2)#74,(sat_sales1 / sat_sales2)#75]), [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75], 100, 0, [s_store_name1#28 ASC NULLS FIRST, s_store_id1#30 ASC NULLS FIRST, d_week_seq1#29 ASC NULLS FIRST], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75] -(39) ColumnarToRow [codegen id : 1] -Input [10]: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] +(42) CometColumnarToRow [codegen id : 1] +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/extended.txt new file mode 100644 index 0000000000..bfd467b72b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/simplified.txt index e95d05323f..72823b14c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] @@ -8,34 +8,37 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name,d_week_seq] CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id,s_store_name] - CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] CometExchange [d_week_seq,ss_store_sk] #1 - CometHashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] + CometHashAggregate [d_day_name,ss_sales_price] [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id] - CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [d_week_seq] #7 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/explain.txt index 36540b4276..46bd113463 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/explain.txt @@ -1,243 +1,274 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometFilter (36) - +- CometHashAggregate (35) - +- CometExchange (34) - +- CometHashAggregate (33) - +- CometProject (32) - +- CometBroadcastHashJoin (31) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`item` (21) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer_address` +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometNativeScan parquet spark_catalog.default.item (22) + + +(1) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] -Arguments: [ca_address_sk#1, ca_state#2] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] Condition : isnotnull(ca_address_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(4) CometFilter -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(6) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#1, ca_state#2] -Right output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_address_sk#1], [c_current_addr_sk#4], Inner, BuildRight - -(7) CometProject -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_state#2, c_customer_sk#3], [ca_state#2, c_customer_sk#3] - -(8) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(9) CometFilter -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) - -(10) CometBroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(11) CometBroadcastHashJoin -Left output [2]: [ca_state#2, c_customer_sk#3] -Right output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], Inner, BuildRight - -(12) CometProject -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7], [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8, d_month_seq#9] - -(14) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(3) CometProject +Input [2]: [ca_address_sk#1, ca_state#2] +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] + +(4) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) + +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] + +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight + +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] + +(9) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) + +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] + +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight + +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) + +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] + +(20) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) + +(22) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_current_price#16, i_category#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(23) CometFilter +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true)) + +(24) CometProject +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_current_price#16, i_category#18], [i_current_price#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#18] -(15) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +(25) CometHashAggregate +Input [2]: [i_current_price#16, i_category#18] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] -(16) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +(26) CometExchange +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) CometBroadcastHashJoin -Left output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(27) CometHashAggregate +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] -(18) CometProject -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ca_state#2, ss_item_sk#5], [ca_state#2, ss_item_sk#5] +(28) CometFilter +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Arguments: [i_item_sk#12, i_current_price#13, i_category#14] +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] -(20) CometFilter -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight -(21) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_current_price#15, i_category#16] -Arguments: [i_current_price#15, i_category#16] +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] -(22) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] -(23) CometHashAggregate -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight -(24) CometExchange -Input [3]: [i_category#16, sum#17, count#18] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] -(25) CometHashAggregate -Input [3]: [i_category#16, sum#17, count#18] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] - -(26) CometFilter -Input [2]: [avg(i_current_price)#19, i_category#16] -Condition : isnotnull(avg(i_current_price)#19) - -(27) CometBroadcastExchange -Input [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [avg(i_current_price)#19, i_category#16] - -(28) CometBroadcastHashJoin -Left output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Right output [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [i_category#14], [i_category#16], Inner, (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight - -(29) CometProject -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#19, i_category#16] -Arguments: [i_item_sk#12], [i_item_sk#12] - -(30) CometBroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: [i_item_sk#12] - -(31) CometBroadcastHashJoin -Left output [2]: [ca_state#2, ss_item_sk#5] -Right output [1]: [i_item_sk#12] -Arguments: [ss_item_sk#5], [i_item_sk#12], Inner, BuildRight - -(32) CometProject -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] -Arguments: [ca_state#2], [ca_state#2] - -(33) CometHashAggregate -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] -(34) CometExchange -Input [2]: [ca_state#2, count#20] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(36) CometExchange +Input [2]: [ca_state#3, count#22] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) CometHashAggregate -Input [2]: [ca_state#2, count#20] -Keys [1]: [ca_state#2] +(37) CometHashAggregate +Input [2]: [ca_state#3, count#22] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -(36) CometFilter -Input [2]: [state#21, cnt#22] -Condition : (cnt#22 >= 10) +(38) CometFilter +Input [2]: [state#23, cnt#24] +Condition : (cnt#24 >= 10) -(37) CometTakeOrderedAndProject -Input [2]: [state#21, cnt#22] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, [cnt#22 ASC NULLS FIRST], [state#21, cnt#22] +(39) CometTakeOrderedAndProject +Input [2]: [state#23, cnt#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST], [state#23, cnt#24] -(38) ColumnarToRow [codegen id : 1] -Input [2]: [state#21, cnt#22] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (45) -+- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometFilter (40) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23, d_year#24, d_moy#25] +(41) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] +ReadSchema: struct -(40) CometFilter -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(41) CometProject -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23], [d_month_seq#23] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(42) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(43) CometExchange -Input [1]: [d_month_seq#23] -Arguments: hashpartitioning(d_month_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(44) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(45) ColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#23] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/extended.txt new file mode 100644 index 0000000000..f6467c6fd9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/extended.txt @@ -0,0 +1,50 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 45 out of 46 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/simplified.txt index 4ea38af90f..3e74bb125f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [state,cnt] CometFilter [state,cnt] - CometHashAggregate [state,cnt,ca_state,count,count(1)] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] @@ -14,37 +14,39 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometProject [ca_state,c_customer_sk] CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [d_month_seq] CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] - CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] CometExchange [i_category] #8 - CometHashAggregate [i_category,sum,count,i_current_price] - CometFilter [i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_current_price,i_category] + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/explain.txt index e5ada56e73..51f98a2de9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/explain.txt @@ -1,66 +1,86 @@ == Physical Plan == -* ColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometHashAggregate (47) - +- CometExchange (46) - +- CometHashAggregate (45) - +- CometUnion (44) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- CometHashAggregate (43) - +- ReusedExchange (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] @@ -83,9 +103,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] @@ -108,151 +131,218 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometNativeScan: `spark_catalog`.`default`.`item` +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12, i_category#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct (18) CometFilter Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Music ) (19) CometProject Input [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12], [i_item_id#12] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(22) CometBroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] -(23) CometBroadcastHashJoin +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] +Right output [2]: [i_item_sk#10, i_item_id#15] Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometExchange -Input [2]: [i_item_id#11, sum#14] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [2]: [i_item_id#11, sum#14] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(29) CometFilter -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) - -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct -(31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) -(32) CometProject -Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#20] +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight -(34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] -Right output [1]: [ca_address_sk#20] -Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] -(35) CometProject -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] -Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] -(36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#21, i_item_id#22] +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight -(37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] -Right output [2]: [i_item_sk#21, i_item_id#22] -Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] -(38) CometProject -Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_item_id#22] -Arguments: [cs_ext_sales_price#17, i_item_id#22], [cs_ext_sales_price#17, i_item_id#22] +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] -(39) CometHashAggregate -Input [2]: [cs_ext_sales_price#17, i_item_id#22] -Keys [1]: [i_item_id#22] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight -(40) CometExchange -Input [2]: [i_item_id#22, sum#23] -Arguments: hashpartitioning(i_item_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] -(41) CometHashAggregate -Input [2]: [i_item_id#22, sum#23] -Keys [1]: [i_item_id#22] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] -(42) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_item_id#24, sum#25] +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(43) CometHashAggregate +(42) CometHashAggregate Input [2]: [i_item_id#24, sum#25] Keys [1]: [i_item_id#24] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] - -(44) CometUnion -Child 0 Input [2]: [i_item_id#11, total_sales#27] -Child 1 Input [2]: [i_item_id#22, total_sales#28] -Child 2 Input [2]: [i_item_id#24, total_sales#29] - -(45) CometHashAggregate -Input [2]: [i_item_id#11, total_sales#27] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#27)] - -(46) CometExchange -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(47) CometHashAggregate -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#27)] - -(48) CometTakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#11 ASC NULLS FIRST,total_sales#32 ASC NULLS FIRST], output=[i_item_id#11,total_sales#32]), [i_item_id#11, total_sales#32], 100, [i_item_id#11 ASC NULLS FIRST, total_sales#32 ASC NULLS FIRST], [i_item_id#11, total_sales#32] - -(49) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#11, total_sales#32] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] + +(43) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#35 ASC NULLS FIRST,total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [i_item_id#35 ASC NULLS FIRST, total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/extended.txt new file mode 100644 index 0000000000..7d3fb6fd9a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/extended.txt @@ -0,0 +1,92 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 89 out of 89 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/simplified.txt index 5bc8aaae31..99304a1eed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,isEmpty,total_sales] + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] CometUnion [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id] #2 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_ext_sales_price] @@ -16,26 +16,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_category] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_category] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #7 - CometHashAggregate [i_item_id,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_ext_sales_price] @@ -43,9 +44,21 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [i_item_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/explain.txt index 75c3a8e9c6..bd3852145d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/explain.txt @@ -1,14 +1,14 @@ == Physical Plan == -* Project (62) -+- * BroadcastNestedLoopJoin Inner BuildRight (61) - :- * ColumnarToRow (38) - : +- CometHashAggregate (37) - : +- CometExchange (36) - : +- CometHashAggregate (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) +* Project (65) ++- * BroadcastNestedLoopJoin Inner BuildRight (64) + :- * CometColumnarToRow (41) + : +- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) : : :- CometProject (25) : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) @@ -18,62 +18,72 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (15) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (21) - : : +- ReusedExchange (26) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (29) - +- BroadcastExchange (60) - +- * ColumnarToRow (59) - +- CometHashAggregate (58) - +- CometExchange (57) - +- CometHashAggregate (56) - +- CometProject (55) - +- CometBroadcastHashJoin (54) - :- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (49) - : : +- CometBroadcastHashJoin (48) - : : :- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometFilter (40) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (39) - : : : : +- ReusedExchange (41) - : : : +- ReusedExchange (44) - : : +- ReusedExchange (47) - : +- ReusedExchange (50) - +- ReusedExchange (53) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.customer (21) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.item (32) + +- BroadcastExchange (63) + +- * CometColumnarToRow (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometProject (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometProject (49) + : : : +- CometBroadcastHashJoin (48) + : : : :- CometProject (46) + : : : : +- CometBroadcastHashJoin (45) + : : : : :- CometFilter (43) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (42) + : : : : +- ReusedExchange (44) + : : : +- ReusedExchange (47) + : : +- ReusedExchange (50) + : +- ReusedExchange (53) + +- ReusedExchange (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`store` +(3) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#7, s_gmt_offset#8] -Arguments: [s_store_sk#7, s_gmt_offset#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [s_store_sk#7, s_gmt_offset#8] @@ -96,13 +106,16 @@ Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) CometNativeScan: `spark_catalog`.`default`.`promotion` +(9) CometNativeScan parquet spark_catalog.default.promotion Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Arguments: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#10, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#11, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#12, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#9)) (11) CometProject Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] @@ -121,9 +134,12 @@ Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -(15) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(15) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_year#14, d_moy#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] @@ -146,9 +162,12 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -(21) CometNativeScan: `spark_catalog`.`default`.`customer` +(21) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#16, c_current_addr_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (22) CometFilter Input [2]: [c_customer_sk#16, c_current_addr_sk#17] @@ -167,154 +186,177 @@ Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -(26) ReusedExchange [Reuses operator id: 6] -Output [1]: [ca_address_sk#18] +(26) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#18, ca_gmt_offset#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) + +(28) CometProject +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] + +(29) CometBroadcastExchange +Input [1]: [ca_address_sk#18] +Arguments: [ca_address_sk#18] -(27) CometBroadcastHashJoin +(30) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] Right output [1]: [ca_address_sk#18] Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight -(28) CometProject +(31) CometProject Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] -(29) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19, i_category#20] +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#20, i_category#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(30) CometFilter -Input [2]: [i_item_sk#19, i_category#20] -Condition : ((isnotnull(i_category#20) AND (i_category#20 = Jewelry )) AND isnotnull(i_item_sk#19)) +(33) CometFilter +Input [2]: [i_item_sk#20, i_category#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#21, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#20)) -(31) CometProject -Input [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +(34) CometProject +Input [2]: [i_item_sk#20, i_category#21] +Arguments: [i_item_sk#20], [i_item_sk#20] -(32) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +(35) CometBroadcastExchange +Input [1]: [i_item_sk#20] +Arguments: [i_item_sk#20] -(33) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight -(34) CometProject -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#19] +(37) CometProject +Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] -(35) CometHashAggregate +(38) CometHashAggregate Input [1]: [ss_ext_sales_price#5] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(36) CometExchange -Input [1]: [sum#21] +(39) CometExchange +Input [1]: [sum#22] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(37) CometHashAggregate -Input [1]: [sum#21] +(40) CometHashAggregate +Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -(38) ColumnarToRow [codegen id : 2] -Input [1]: [promotions#22] - -(39) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] -Arguments: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] - -(40) CometFilter -Input [5]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] -Condition : ((isnotnull(ss_store_sk#25) AND isnotnull(ss_customer_sk#24)) AND isnotnull(ss_item_sk#23)) +(41) CometColumnarToRow [codegen id : 2] +Input [1]: [promotions#23] -(41) ReusedExchange [Reuses operator id: 6] -Output [1]: [s_store_sk#28] +(42) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#28)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct -(42) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] -Right output [1]: [s_store_sk#28] -Arguments: [ss_store_sk#25], [s_store_sk#28], Inner, BuildRight +(43) CometFilter +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Condition : ((isnotnull(ss_store_sk#26) AND isnotnull(ss_customer_sk#25)) AND isnotnull(ss_item_sk#24)) -(43) CometProject -Input [6]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, s_store_sk#28] -Arguments: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27], [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27] - -(44) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#29] +(44) ReusedExchange [Reuses operator id: 6] +Output [1]: [s_store_sk#29] (45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27] -Right output [1]: [d_date_sk#29] -Arguments: [ss_sold_date_sk#27], [d_date_sk#29], Inner, BuildRight +Left output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [s_store_sk#29] +Arguments: [ss_store_sk#26], [s_store_sk#29], Inner, BuildRight (46) CometProject -Input [5]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#29] -Arguments: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26], [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26] +Input [6]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28, s_store_sk#29] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] -(47) ReusedExchange [Reuses operator id: 23] -Output [2]: [c_customer_sk#30, c_current_addr_sk#31] +(47) ReusedExchange [Reuses operator id: 18] +Output [1]: [d_date_sk#30] (48) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26] -Right output [2]: [c_customer_sk#30, c_current_addr_sk#31] -Arguments: [ss_customer_sk#24], [c_customer_sk#30], Inner, BuildRight +Left output [4]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [d_date_sk#30] +Arguments: [ss_sold_date_sk#28], [d_date_sk#30], Inner, BuildRight (49) CometProject -Input [5]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, c_customer_sk#30, c_current_addr_sk#31] -Arguments: [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31], [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31] +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28, d_date_sk#30] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] -(50) ReusedExchange [Reuses operator id: 6] -Output [1]: [ca_address_sk#32] +(50) ReusedExchange [Reuses operator id: 23] +Output [2]: [c_customer_sk#31, c_current_addr_sk#32] (51) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31] -Right output [1]: [ca_address_sk#32] -Arguments: [c_current_addr_sk#31], [ca_address_sk#32], Inner, BuildRight +Left output [3]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] +Right output [2]: [c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_customer_sk#25], [c_customer_sk#31], Inner, BuildRight (52) CometProject -Input [4]: [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31, ca_address_sk#32] -Arguments: [ss_item_sk#23, ss_ext_sales_price#26], [ss_item_sk#23, ss_ext_sales_price#26] +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32], [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] -(53) ReusedExchange [Reuses operator id: 32] -Output [1]: [i_item_sk#33] +(53) ReusedExchange [Reuses operator id: 29] +Output [1]: [ca_address_sk#33] (54) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#23, ss_ext_sales_price#26] -Right output [1]: [i_item_sk#33] -Arguments: [ss_item_sk#23], [i_item_sk#33], Inner, BuildRight +Left output [3]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] +Right output [1]: [ca_address_sk#33] +Arguments: [c_current_addr_sk#32], [ca_address_sk#33], Inner, BuildRight (55) CometProject -Input [3]: [ss_item_sk#23, ss_ext_sales_price#26, i_item_sk#33] -Arguments: [ss_ext_sales_price#26], [ss_ext_sales_price#26] +Input [4]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32, ca_address_sk#33] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27], [ss_item_sk#24, ss_ext_sales_price#27] + +(56) ReusedExchange [Reuses operator id: 35] +Output [1]: [i_item_sk#34] + +(57) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#24, ss_ext_sales_price#27] +Right output [1]: [i_item_sk#34] +Arguments: [ss_item_sk#24], [i_item_sk#34], Inner, BuildRight + +(58) CometProject +Input [3]: [ss_item_sk#24, ss_ext_sales_price#27, i_item_sk#34] +Arguments: [ss_ext_sales_price#27], [ss_ext_sales_price#27] -(56) CometHashAggregate -Input [1]: [ss_ext_sales_price#26] +(59) CometHashAggregate +Input [1]: [ss_ext_sales_price#27] Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#26))] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))] -(57) CometExchange -Input [1]: [sum#34] +(60) CometExchange +Input [1]: [sum#35] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(58) CometHashAggregate -Input [1]: [sum#34] +(61) CometHashAggregate +Input [1]: [sum#35] Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#26))] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] -(59) ColumnarToRow [codegen id : 1] -Input [1]: [total#35] +(62) CometColumnarToRow [codegen id : 1] +Input [1]: [total#36] -(60) BroadcastExchange -Input [1]: [total#35] +(63) BroadcastExchange +Input [1]: [total#36] Arguments: IdentityBroadcastMode, [plan_id=3] -(61) BroadcastNestedLoopJoin [codegen id : 2] +(64) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(62) Project [codegen id : 2] -Output [3]: [promotions#22, total#35, ((cast(promotions#22 as decimal(15,4)) / cast(total#35 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#36] -Input [2]: [promotions#22, total#35] +(65) Project [codegen id : 2] +Output [3]: [promotions#23, total#36, ((cast(promotions#23 as decimal(15,4)) / cast(total#36 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#37] +Input [2]: [promotions#23, total#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/extended.txt new file mode 100644 index 0000000000..d3db57c145 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/extended.txt @@ -0,0 +1,81 @@ +Project ++- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 74 out of 77 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/simplified.txt index f2bc0148e2..d5eeeeebc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (2) Project [promotions,total] BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [promotions,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [promotions,sum(UnscaledValue(ss_ext_sales_price))] CometExchange #1 - CometHashAggregate [sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [sum] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] CometProject [ss_item_sk,ss_ext_sales_price] @@ -19,35 +19,38 @@ WholeStageCodegen (2) CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk] #2 CometProject [s_store_sk] CometFilter [s_store_sk,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] CometBroadcastExchange [p_promo_sk] #3 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - ReusedExchange [ca_address_sk] #2 - CometBroadcastExchange [i_item_sk] #6 + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter - BroadcastExchange #7 + BroadcastExchange #8 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [total,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange #8 - CometHashAggregate [sum,ss_ext_sales_price] + CometHashAggregate [sum] [total,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange #9 + CometHashAggregate [ss_ext_sales_price] [sum] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] CometProject [ss_item_sk,ss_ext_sales_price] @@ -59,9 +62,9 @@ WholeStageCodegen (2) CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [s_store_sk] #2 ReusedExchange [d_date_sk] #4 ReusedExchange [c_customer_sk,c_current_addr_sk] #5 - ReusedExchange [ca_address_sk] #2 - ReusedExchange [i_item_sk] #6 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/explain.txt index d555809a8e..1d9748055e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/explain.txt @@ -1,41 +1,52 @@ == Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.web_site (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -54,73 +65,104 @@ Arguments: [ws_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -(8) ReusedExchange [Reuses operator id: 5] +(8) CometNativeScan parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct -(9) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight +(9) CometFilter +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) (10) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [web_site_sk#10, web_name#11] +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] (12) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [web_site_sk#10, web_name#11] -Arguments: [ws_web_site_sk#2], [web_site_sk#10], Inner, BuildRight +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight (13) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] +(14) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#11, web_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [ws_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) CometHashAggregate -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +Input [2]: [web_site_sk#11, web_name#12] +Condition : isnotnull(web_site_sk#11) + +(16) CometBroadcastExchange +Input [2]: [web_site_sk#11, web_name#12] +Arguments: [web_site_sk#11, web_name#12] + +(17) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [web_site_sk#11, web_name#12] +Arguments: [ws_web_site_sk#2], [web_site_sk#11], Inner, BuildRight + +(18) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_site_sk#11, web_name#12] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] + +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) + +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(23) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight + +(24) CometProject +Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12, d_date_sk#13] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] + +(25) CometHashAggregate +Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(21) CometExchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, web_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(23) CometTakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST,sm_type#9 ASC NULLS FIRST,web_name#11 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#20,sm_type#9,web_name#11,30 days #21,31 - 60 days #22,61 - 90 days #23,91 - 120 days #24,>120 days #25]), [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25], 100, [substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,web_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,web_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, web_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] -(24) ColumnarToRow [codegen id : 1] -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/extended.txt new file mode 100644 index 0000000000..52d7828b06 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.ship_mode + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/simplified.txt index d7ce3a3184..de80c17349 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] CometExchange [_groupingexpression,sm_type,web_name] #1 - CometHashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum,ws_ship_date_sk,ws_sold_date_sk] + CometHashAggregate [ws_ship_date_sk,ws_sold_date_sk] [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] @@ -14,13 +14,18 @@ WholeStageCodegen (1) CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [web_site_sk,web_name] #2 - CometBroadcastExchange [d_date_sk] #3 + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [web_site_sk,web_name] #4 + CometFilter [web_site_sk,web_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_name] + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/explain.txt index 455002dee6..a1e7286cb0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/explain.txt @@ -3,7 +3,7 @@ TakeOrderedAndProject (29) +- * Project (28) +- * Filter (27) +- Window (26) - +- * ColumnarToRow (25) + +- * CometColumnarToRow (25) +- CometSort (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -17,34 +17,41 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) + +- CometNativeScan parquet spark_catalog.default.store (15) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,refernece ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) (3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -63,9 +70,12 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_moy#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] @@ -88,9 +98,12 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [1]: [s_store_sk#17] @@ -131,7 +144,7 @@ Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNative Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] Arguments: [i_manager_id#5, sum_sales#19, _w0#20], [i_manager_id#5 ASC NULLS FIRST] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] (26) Window diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/extended.txt new file mode 100644 index 0000000000..44fd48691b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/extended.txt @@ -0,0 +1,31 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/simplified.txt index 081555fe51..4a2ef17f8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/simplified.txt @@ -5,13 +5,13 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_manager_id] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_manager_id,sum_sales,_w0] CometExchange [i_manager_id] #1 - CometHashAggregate [i_manager_id,sum_sales,_w0,d_moy,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [d_moy,sum] [i_manager_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_manager_id,d_moy] #2 - CometHashAggregate [i_manager_id,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_manager_id,d_moy,sum] CometProject [i_manager_id,ss_sales_price,d_moy] CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] @@ -20,14 +20,14 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manager_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_month_seq,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/explain.txt index dffc690465..e2d82c6a20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/explain.txt @@ -1,41 +1,41 @@ == Physical Plan == -* ColumnarToRow (163) -+- CometSort (162) - +- CometColumnarExchange (161) - +- CometProject (160) - +- CometSortMergeJoin (159) - :- CometSort (98) - : +- CometExchange (97) - : +- CometHashAggregate (96) - : +- CometHashAggregate (95) - : +- CometProject (94) - : +- CometBroadcastHashJoin (93) - : :- CometProject (88) - : : +- CometBroadcastHashJoin (87) - : : :- CometProject (85) - : : : +- CometBroadcastHashJoin (84) - : : : :- CometProject (82) - : : : : +- CometBroadcastHashJoin (81) - : : : : :- CometProject (79) - : : : : : +- CometBroadcastHashJoin (78) - : : : : : :- CometProject (74) - : : : : : : +- CometBroadcastHashJoin (73) - : : : : : : :- CometProject (71) - : : : : : : : +- CometBroadcastHashJoin (70) - : : : : : : : :- CometProject (66) - : : : : : : : : +- CometBroadcastHashJoin (65) - : : : : : : : : :- CometProject (61) - : : : : : : : : : +- CometBroadcastHashJoin (60) - : : : : : : : : : :- CometProject (58) - : : : : : : : : : : +- CometBroadcastHashJoin (57) - : : : : : : : : : : :- CometProject (53) - : : : : : : : : : : : +- CometBroadcastHashJoin (52) - : : : : : : : : : : : :- CometProject (50) - : : : : : : : : : : : : +- CometBroadcastHashJoin (49) - : : : : : : : : : : : : :- CometProject (45) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (44) - : : : : : : : : : : : : : :- CometProject (40) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (39) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) : : : : : : : : : : : : : : :- CometProject (35) : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) : : : : : : : : : : : : : : : :- CometProject (30) @@ -46,10 +46,10 @@ : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,111 +62,120 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (11) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (31) - : : : : : : : : : : : : : : +- CometBroadcastExchange (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (36) - : : : : : : : : : : : : : +- CometBroadcastExchange (43) - : : : : : : : : : : : : : +- CometFilter (42) - : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (41) - : : : : : : : : : : : : +- CometBroadcastExchange (48) - : : : : : : : : : : : : +- CometFilter (47) - : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (46) - : : : : : : : : : : : +- ReusedExchange (51) - : : : : : : : : : : +- CometBroadcastExchange (56) - : : : : : : : : : : +- CometFilter (55) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (54) - : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : +- CometBroadcastExchange (64) - : : : : : : : : +- CometFilter (63) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (62) - : : : : : : : +- CometBroadcastExchange (69) - : : : : : : : +- CometFilter (68) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (67) - : : : : : : +- ReusedExchange (72) - : : : : : +- CometBroadcastExchange (77) - : : : : : +- CometFilter (76) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (75) - : : : : +- ReusedExchange (80) - : : : +- ReusedExchange (83) - : : +- ReusedExchange (86) - : +- CometBroadcastExchange (92) - : +- CometProject (91) - : +- CometFilter (90) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (89) - +- CometSort (158) - +- CometExchange (157) - +- CometHashAggregate (156) - +- CometHashAggregate (155) - +- CometProject (154) - +- CometBroadcastHashJoin (153) - :- CometProject (151) - : +- CometBroadcastHashJoin (150) - : :- CometProject (148) - : : +- CometBroadcastHashJoin (147) - : : :- CometProject (145) - : : : +- CometBroadcastHashJoin (144) - : : : :- CometProject (142) - : : : : +- CometBroadcastHashJoin (141) - : : : : :- CometProject (139) - : : : : : +- CometBroadcastHashJoin (138) - : : : : : :- CometProject (136) - : : : : : : +- CometBroadcastHashJoin (135) - : : : : : : :- CometProject (133) - : : : : : : : +- CometBroadcastHashJoin (132) - : : : : : : : :- CometProject (130) - : : : : : : : : +- CometBroadcastHashJoin (129) - : : : : : : : : :- CometProject (127) - : : : : : : : : : +- CometBroadcastHashJoin (126) - : : : : : : : : : :- CometProject (124) - : : : : : : : : : : +- CometBroadcastHashJoin (123) - : : : : : : : : : : :- CometProject (121) - : : : : : : : : : : : +- CometBroadcastHashJoin (120) - : : : : : : : : : : : :- CometProject (118) - : : : : : : : : : : : : +- CometBroadcastHashJoin (117) - : : : : : : : : : : : : :- CometProject (115) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (114) - : : : : : : : : : : : : : :- CometProject (112) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (111) - : : : : : : : : : : : : : : :- CometProject (107) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (106) - : : : : : : : : : : : : : : : :- CometSort (100) - : : : : : : : : : : : : : : : : +- ReusedExchange (99) - : : : : : : : : : : : : : : : +- CometSort (105) - : : : : : : : : : : : : : : : +- CometProject (104) - : : : : : : : : : : : : : : : +- CometFilter (103) - : : : : : : : : : : : : : : : +- CometHashAggregate (102) - : : : : : : : : : : : : : : : +- ReusedExchange (101) - : : : : : : : : : : : : : : +- CometBroadcastExchange (110) - : : : : : : : : : : : : : : +- CometFilter (109) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (108) - : : : : : : : : : : : : : +- ReusedExchange (113) - : : : : : : : : : : : : +- ReusedExchange (116) - : : : : : : : : : : : +- ReusedExchange (119) - : : : : : : : : : : +- ReusedExchange (122) - : : : : : : : : : +- ReusedExchange (125) - : : : : : : : : +- ReusedExchange (128) - : : : : : : : +- ReusedExchange (131) - : : : : : : +- ReusedExchange (134) - : : : : : +- ReusedExchange (137) - : : : : +- ReusedExchange (140) - : : : +- ReusedExchange (143) - : : +- ReusedExchange (146) - : +- ReusedExchange (149) - +- ReusedExchange (152) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometNativeScan parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] +ReadSchema: struct (2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -176,9 +185,12 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (5) CometFilter Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] @@ -205,9 +217,12 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(11) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct (12) CometFilter Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] @@ -225,9 +240,12 @@ Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct (17) CometFilter Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] @@ -289,9 +307,12 @@ Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#31, d_year#32] -Arguments: [d_date_sk#31, d_year#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (32) CometFilter Input [2]: [d_date_sk#31, d_year#32] @@ -310,528 +331,579 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(36) CometNativeScan: `spark_catalog`.`default`.`store` +(36) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] +ReadSchema: struct (37) CometFilter Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(s_zip#35)) +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(38) CometBroadcastExchange +(38) CometProject Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] + +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] -(39) CometBroadcastHashJoin +(40) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(40) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] - -(41) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(42) CometFilter -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Condition : (((((isnotnull(c_customer_sk#36) AND isnotnull(c_first_sales_date_sk#41)) AND isnotnull(c_first_shipto_date_sk#40)) AND isnotnull(c_current_cdemo_sk#37)) AND isnotnull(c_current_hdemo_sk#38)) AND isnotnull(c_current_addr_sk#39)) - -(43) CometBroadcastExchange -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(44) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] -Right output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_customer_sk#2], [c_customer_sk#36], Inner, BuildRight - -(45) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(46) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(47) CometFilter -Input [2]: [d_date_sk#42, d_year#43] -Condition : isnotnull(d_date_sk#42) - -(48) CometBroadcastExchange -Input [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(49) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Right output [2]: [d_date_sk#42, d_year#43] -Arguments: [c_first_sales_date_sk#41], [d_date_sk#42], Inner, BuildRight - -(50) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41, d_date_sk#42, d_year#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] - -(51) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#44, d_year#45] - -(52) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_shipto_date_sk#40], [d_date_sk#44], Inner, BuildRight - -(53) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(54) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(55) CometFilter -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Condition : (isnotnull(cd_demo_sk#46) AND isnotnull(cd_marital_status#47)) - -(56) CometBroadcastExchange -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(57) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#46], Inner, BuildRight - -(58) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] - -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#48, cd_marital_status#49] - -(60) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [c_current_cdemo_sk#37], [cd_demo_sk#48], Inner, NOT (cd_marital_status#47 = cd_marital_status#49), BuildRight - -(61) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47, cd_demo_sk#48, cd_marital_status#49] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(62) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(63) CometFilter -Input [1]: [p_promo_sk#50] -Condition : isnotnull(p_promo_sk#50) - -(64) CometBroadcastExchange -Input [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(65) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [1]: [p_promo_sk#50] -Arguments: [ss_promo_sk#7], [p_promo_sk#50], Inner, BuildRight - -(66) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, p_promo_sk#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(67) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] - -(68) CometFilter -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Condition : (isnotnull(hd_demo_sk#51) AND isnotnull(hd_income_band_sk#52)) - -(69) CometBroadcastExchange -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] + +(42) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) + +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] + +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight + +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] + +(47) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) + +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] + +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight + +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] + +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] + +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight + +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] + +(55) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) + +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] + +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] + +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight + +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] + +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] + +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] + +(64) CometNativeScan parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) + +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] + +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight + +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] + +(69) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct -(70) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#51], Inner, BuildRight +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(71) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(72) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] - -(73) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [c_current_hdemo_sk#38], [hd_demo_sk#53], Inner, BuildRight +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight + +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(74) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(75) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(76) CometFilter -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Condition : isnotnull(ca_address_sk#55) +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(77) CometBroadcastExchange -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +(77) CometNativeScan parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(78) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] -Right output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_addr_sk#5], [ca_address_sk#55], Inner, BuildRight +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(80) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Right output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [c_current_addr_sk#39], [ca_address_sk#60], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(83) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#65] +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (84) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#65] -Arguments: [hd_income_band_sk#52], [ib_income_band_sk#65], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight (85) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#65] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(86) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#66] - -(87) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#66] -Arguments: [hd_income_band_sk#54], [ib_income_band_sk#66], Inner, BuildRight - -(88) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#66] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(89) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] - -(90) CometFilter -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Condition : ((((((isnotnull(i_current_price#68) AND i_color#69 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#68 >= 64.00)) AND (i_current_price#68 <= 74.00)) AND (i_current_price#68 >= 65.00)) AND (i_current_price#68 <= 79.00)) AND isnotnull(i_item_sk#67)) - -(91) CometProject -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70], [i_item_sk#67, i_product_name#70] - -(92) CometBroadcastExchange -Input [2]: [i_item_sk#67, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70] - -(93) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [2]: [i_item_sk#67, i_product_name#70] -Arguments: [ss_item_sk#1], [i_item_sk#67], Inner, BuildRight - -(94) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] - -(95) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(86) CometNativeScan parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) + +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] + +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight + +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] + +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight + +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(94) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) + +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] + +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] + +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight + +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] + +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -(96) CometHashAggregate -Input [19]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45, count#71, sum#72, sum#73, sum#74] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(97) CometExchange -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: hashpartitioning(item_sk#76, store_name#77, store_zip#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(98) CometSort -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91], [item_sk#76 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, store_zip#78 ASC NULLS FIRST] - -(99) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(100) CometSort -Input [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92 ASC NULLS FIRST] +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(101) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] - -(102) CometHashAggregate -Input [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] -Keys [1]: [cs_item_sk#103] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#107)), sum(((cr_refunded_cash#108 + cr_reversed_charge#109) + cr_store_credit#110))] - -(103) CometFilter -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) - -(104) CometProject -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Arguments: [cs_item_sk#103], [cs_item_sk#103] +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] (105) CometSort -Input [1]: [cs_item_sk#103] -Arguments: [cs_item_sk#103], [cs_item_sk#103 ASC NULLS FIRST] +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(106) CometSortMergeJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [1]: [cs_item_sk#103] -Arguments: [ss_item_sk#92], [cs_item_sk#103], Inner +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(107) CometProject -Input [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, cs_item_sk#103] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] -(108) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(109) CometFilter -Input [2]: [d_date_sk#111, d_year#112] -Condition : ((isnotnull(d_year#112) AND (d_year#112 = 2000)) AND isnotnull(d_date_sk#111)) +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(110) CometBroadcastExchange -Input [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(111) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ss_sold_date_sk#102], [d_date_sk#111], Inner, BuildRight +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner (112) CometProject -Input [13]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, d_date_sk#111, d_year#112] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] - -(113) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] - -(114) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] -Right output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_store_sk#97], [s_store_sk#113], Inner, BuildRight - -(115) CometProject -Input [14]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] - -(116) ReusedExchange [Reuses operator id: 43] -Output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(117) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] -Right output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_customer_sk#93], [c_customer_sk#116], Inner, BuildRight +(113) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) + +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] + +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight + +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] + +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] + +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight + +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] + +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight + +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] + +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight + +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] + +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] + +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight + +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] + +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight + +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] + +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] + +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] -(118) CometProject -Input [18]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight -(119) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#122, d_year#123] +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(120) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [c_first_sales_date_sk#121], [d_date_sk#122], Inner, BuildRight - -(121) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] - -(122) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#124, d_year#125] - -(123) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [c_first_shipto_date_sk#120], [d_date_sk#124], Inner, BuildRight - -(124) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123, d_date_sk#124, d_year#125] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(125) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#126, cd_marital_status#127] - -(126) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_cdemo_sk#94], [cd_demo_sk#126], Inner, BuildRight - -(127) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] - -(128) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#128, cd_marital_status#129] - -(129) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] -Right output [2]: [cd_demo_sk#128, cd_marital_status#129] -Arguments: [c_current_cdemo_sk#117], [cd_demo_sk#128], Inner, NOT (cd_marital_status#127 = cd_marital_status#129), BuildRight - -(130) CometProject -Input [18]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127, cd_demo_sk#128, cd_marital_status#129] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(131) ReusedExchange [Reuses operator id: 64] -Output [1]: [p_promo_sk#130] - -(132) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [1]: [p_promo_sk#130] -Arguments: [ss_promo_sk#98], [p_promo_sk#130], Inner, BuildRight - -(133) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, p_promo_sk#130] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(134) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#131, hd_income_band_sk#132] +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -(135) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_hdemo_sk#95], [hd_demo_sk#131], Inner, BuildRight +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight -(136) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(137) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] -(138) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] -Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [c_current_hdemo_sk#118], [hd_demo_sk#133], Inner, BuildRight +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight -(139) CometProject -Input [15]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] -(140) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(141) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] -Right output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_addr_sk#96], [ca_address_sk#135], Inner, BuildRight +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight -(142) CometProject -Input [18]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(143) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(144) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Right output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [c_current_addr_sk#119], [ca_address_sk#140], Inner, BuildRight +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight -(145) CometProject -Input [21]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(146) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#145] +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] -(147) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#145] -Arguments: [hd_income_band_sk#132], [ib_income_band_sk#145], Inner, BuildRight +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight -(148) CometProject -Input [20]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#145] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(149) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#146] +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] -(150) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#146] -Arguments: [hd_income_band_sk#134], [ib_income_band_sk#146], Inner, BuildRight +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight -(151) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#146] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(152) ReusedExchange [Reuses operator id: 92] -Output [2]: [i_item_sk#147, i_product_name#148] +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] -(153) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [2]: [i_item_sk#147, i_product_name#148] -Arguments: [ss_item_sk#92], [i_item_sk#147], Inner, BuildRight +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight -(154) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Arguments: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148], [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] -(155) CometHashAggregate -Input [18]: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#99)), partial_sum(UnscaledValue(ss_list_price#100)), partial_sum(UnscaledValue(ss_coupon_amt#101))] +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] -(156) CometHashAggregate -Input [19]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125, count#71, sum#149, sum#150, sum#151] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#99)), sum(UnscaledValue(ss_list_price#100)), sum(UnscaledValue(ss_coupon_amt#101))] +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] -(157) CometExchange -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: hashpartitioning(item_sk#152, store_name#153, store_zip#154, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(158) CometSort -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159], [item_sk#152 ASC NULLS FIRST, store_name#153 ASC NULLS FIRST, store_zip#154 ASC NULLS FIRST] +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] -(159) CometSortMergeJoin -Left output [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Right output [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#76, store_name#77, store_zip#78], [item_sk#152, store_name#153, store_zip#154], Inner, (cnt#156 <= cnt#88) +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) -(160) CometProject -Input [25]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] -(161) CometColumnarExchange -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: rangepartitioning(product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(162) CometSort -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST] +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST] -(163) ColumnarToRow [codegen id : 1] -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/extended.txt new file mode 100644 index 0000000000..ae10068c4c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/extended.txt @@ -0,0 +1,237 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometProject + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.income_band + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 234 out of 234 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/simplified.txt index ef0fff5d7a..4451988479 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt] #1 + CometExchange [product_name,store_name,cnt] #1 CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #2 - CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -47,66 +47,71 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] CometExchange [cs_item_sk] #5 - CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #11 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #12 CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - CometBroadcastExchange [i_item_sk,i_product_name] #16 - CometProject [i_item_sk,i_product_name] + CometBroadcastExchange [ib_income_band_sk] #16 + CometFilter [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 + CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #17 - CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometExchange [item_sk,store_name,store_zip] #18 + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -144,11 +149,11 @@ WholeStageCodegen (1) CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - CometBroadcastExchange [d_date_sk,d_year] #18 + CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #9 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 ReusedExchange [d_date_sk,d_year] #11 @@ -160,6 +165,6 @@ WholeStageCodegen (1) ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [i_item_sk,i_product_name] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/explain.txt index 329ce1b5f2..c72b9a5e75 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/explain.txt @@ -1,14 +1,14 @@ == Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (1) + : : : +- CometNativeScan parquet spark_catalog.default.store (1) : : +- CometBroadcastExchange (15) : : +- CometFilter (14) : : +- CometHashAggregate (13) @@ -17,48 +17,59 @@ : : +- CometProject (10) : : +- CometBroadcastHashJoin (9) : : :- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (3) : : +- CometBroadcastExchange (8) : : +- CometProject (7) : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (5) - : +- CometBroadcastExchange (20) - : +- CometFilter (19) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (18) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometHashAggregate (33) - +- CometExchange (32) - +- CometHashAggregate (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (23) - +- ReusedExchange (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (5) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (36) + +- CometFilter (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.store_sales (24) + +- ReusedExchange (26) + + +(1) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#1, s_store_name#2] -Arguments: [s_store_sk#1, s_store_name#2] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(5) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_month_seq#8] -Arguments: [d_date_sk#7, d_month_seq#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] +ReadSchema: struct (6) CometFilter Input [2]: [d_date_sk#7, d_month_seq#8] @@ -112,96 +123,107 @@ Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] -(18) CometNativeScan: `spark_catalog`.`default`.`item` +(18) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (19) CometFilter Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] Condition : isnotnull(i_item_sk#11) -(20) CometBroadcastExchange +(20) CometProject Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#16] -(21) CometBroadcastHashJoin +(21) CometBroadcastExchange +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(22) CometBroadcastHashJoin Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] -Right output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Right output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] Arguments: [ss_item_sk#3], [i_item_sk#11], Inner, BuildRight -(22) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10, i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15], [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(23) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Arguments: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] +(23) CometProject +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10, i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] -(24) CometFilter -Input [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Condition : isnotnull(ss_store_sk#17) +(24) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct -(25) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] +(25) CometFilter +Input [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_store_sk#18) -(26) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Right output [1]: [d_date_sk#20] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight +(26) ReusedExchange [Reuses operator id: 8] +Output [1]: [d_date_sk#21] -(27) CometProject -Input [5]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19, d_date_sk#20] -Arguments: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18], [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18] +(27) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight -(28) CometHashAggregate -Input [3]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18] -Keys [2]: [ss_store_sk#17, ss_item_sk#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#18))] +(28) CometProject +Input [5]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21] +Arguments: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19], [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] -(29) CometExchange -Input [3]: [ss_store_sk#17, ss_item_sk#16, sum#21] -Arguments: hashpartitioning(ss_store_sk#17, ss_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(29) CometHashAggregate +Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] -(30) CometHashAggregate -Input [3]: [ss_store_sk#17, ss_item_sk#16, sum#21] -Keys [2]: [ss_store_sk#17, ss_item_sk#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#18))] +(30) CometExchange +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate -Input [2]: [ss_store_sk#17, revenue#22] -Keys [1]: [ss_store_sk#17] -Functions [1]: [partial_avg(revenue#22)] - -(32) CometExchange -Input [3]: [ss_store_sk#17, sum#23, count#24] -Arguments: hashpartitioning(ss_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(33) CometHashAggregate -Input [3]: [ss_store_sk#17, sum#23, count#24] -Keys [1]: [ss_store_sk#17] -Functions [1]: [avg(revenue#22)] - -(34) CometFilter -Input [2]: [ss_store_sk#17, ave#25] -Condition : isnotnull(ave#25) - -(35) CometBroadcastExchange -Input [2]: [ss_store_sk#17, ave#25] -Arguments: [ss_store_sk#17, ave#25] - -(36) CometBroadcastHashJoin -Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Right output [2]: [ss_store_sk#17, ave#25] -Arguments: [ss_store_sk#4], [ss_store_sk#17], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#25)), BuildRight - -(37) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15, ss_store_sk#17, ave#25] -Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(38) CometTakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#15]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15], 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(39) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [sum(UnscaledValue(ss_sales_price#19))] + +(32) CometHashAggregate +Input [2]: [ss_store_sk#18, revenue#23] +Keys [1]: [ss_store_sk#18] +Functions [1]: [partial_avg(revenue#23)] + +(33) CometExchange +Input [3]: [ss_store_sk#18, sum#24, count#25] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [3]: [ss_store_sk#18, sum#24, count#25] +Keys [1]: [ss_store_sk#18] +Functions [1]: [avg(revenue#23)] + +(35) CometFilter +Input [2]: [ss_store_sk#18, ave#26] +Condition : isnotnull(ave#26) + +(36) CometBroadcastExchange +Input [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#18, ave#26] + +(37) CometBroadcastHashJoin +Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Right output [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#4], [ss_store_sk#18], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#26)), BuildRight + +(38) CometProject +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16, ss_store_sk#18, ave#26] +Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(39) CometTakeOrderedAndProject +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#16]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(40) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/extended.txt new file mode 100644 index 0000000000..670d8ea5e8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/simplified.txt index 0b9c656364..5ae7059ffe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] CometProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] @@ -9,33 +9,34 @@ WholeStageCodegen (1) CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] - CometHashAggregate [ss_store_sk,ss_item_sk,revenue,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [ss_store_sk,ss_item_sk,revenue,sum(UnscaledValue(ss_sales_price))] CometExchange [ss_store_sk,ss_item_sk] #2 - CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #4 - CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometBroadcastExchange [ss_store_sk,ave] #5 CometFilter [ss_store_sk,ave] - CometHashAggregate [ss_store_sk,ave,sum,count,avg(revenue)] + CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] CometExchange [ss_store_sk] #6 - CometHashAggregate [ss_store_sk,sum,count,revenue] - CometHashAggregate [ss_store_sk,revenue,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [revenue] [ss_store_sk,sum,count] + CometHashAggregate [ss_item_sk,sum] [ss_store_sk,revenue,sum(UnscaledValue(ss_sales_price))] CometExchange [ss_store_sk,ss_item_sk] #7 - CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/explain.txt index 83806709cd..35c81a69d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/explain.txt @@ -1,185 +1,287 @@ == Physical Plan == -* ColumnarToRow (35) -+- CometTakeOrderedAndProject (34) - +- CometHashAggregate (33) - +- CometExchange (32) - +- CometHashAggregate (31) - +- CometUnion (30) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (51) ++- CometTakeOrderedAndProject (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometUnion (46) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (13) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`ship_mode` (19) - +- CometHashAggregate (29) - +- ReusedExchange (28) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.ship_mode (20) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] +PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Condition : isnotnull(w_warehouse_sk#8) -(5) CometBroadcastExchange +(5) CometProject Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#13, 2, true, false, true) AS w_state#15, w_country#14] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] + +(7) CometBroadcastHashJoin Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight -(7) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(9) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(11) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Right output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(13) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18, t_time#19] - -(14) CometFilter -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) - -(15) CometProject -Input [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18], [t_time_sk#18] - -(16) CometBroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: [t_time_sk#18] - -(17) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [t_time_sk#18] -Arguments: [ws_sold_time_sk#1], [t_time_sk#18], Inner, BuildRight - -(18) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(19) CometNativeScan: `spark_catalog`.`default`.`ship_mode` -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20, sm_carrier#21] - -(20) CometFilter -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) - -(21) CometProject -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20], [sm_ship_mode_sk#20] - -(22) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: [sm_ship_mode_sk#20] - -(23) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [sm_ship_mode_sk#20] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#20], Inner, BuildRight - -(24) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(25) CometHashAggregate -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] - -(26) CometExchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] - -(28) ReusedExchange [Reuses operator id: 26] -Output [55]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, d_year#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] - -(29) CometHashAggregate -Input [55]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, d_year#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] -Keys [7]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, d_year#76] -Functions [24]: [sum(CASE WHEN (d_moy#125 = 1) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 2) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 3) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 4) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 5) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 6) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 7) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 8) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 9) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 10) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 11) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 12) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 1) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 2) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 3) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 4) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 5) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 6) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 7) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 8) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 9) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 10) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 11) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 12) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END)] - -(30) CometUnion -Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#131, feb_sales#132, mar_sales#133, apr_sales#134, may_sales#135, jun_sales#136, jul_sales#137, aug_sales#138, sep_sales#139, oct_sales#140, nov_sales#141, dec_sales#142, jan_net#143, feb_net#144, mar_net#145, apr_net#146, may_net#147, jun_net#148, jul_net#149, aug_net#150, sep_net#151, oct_net#152, nov_net#153, dec_net#154] -Child 1 Input [32]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, ship_carriers#155, year#156, jan_sales#157, feb_sales#158, mar_sales#159, apr_sales#160, may_sales#161, jun_sales#162, jul_sales#163, aug_sales#164, sep_sales#165, oct_sales#166, nov_sales#167, dec_sales#168, jan_net#169, feb_net#170, mar_net#171, apr_net#172, may_net#173, jun_net#174, jul_net#175, aug_net#176, sep_net#177, oct_net#178, nov_net#179, dec_net#180] - -(31) CometHashAggregate -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#131, feb_sales#132, mar_sales#133, apr_sales#134, may_sales#135, jun_sales#136, jul_sales#137, aug_sales#138, sep_sales#139, oct_sales#140, nov_sales#141, dec_sales#142, jan_net#143, feb_net#144, mar_net#145, apr_net#146, may_net#147, jun_net#148, jul_net#149, aug_net#150, sep_net#151, oct_net#152, nov_net#153, dec_net#154] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130] -Functions [36]: [partial_sum(jan_sales#131), partial_sum(feb_sales#132), partial_sum(mar_sales#133), partial_sum(apr_sales#134), partial_sum(may_sales#135), partial_sum(jun_sales#136), partial_sum(jul_sales#137), partial_sum(aug_sales#138), partial_sum(sep_sales#139), partial_sum(oct_sales#140), partial_sum(nov_sales#141), partial_sum(dec_sales#142), partial_sum((jan_sales#131 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#132 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#133 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#134 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#135 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#136 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#137 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#138 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#139 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#143), partial_sum(feb_net#144), partial_sum(mar_net#145), partial_sum(apr_net#146), partial_sum(may_net#147), partial_sum(jun_net#148), partial_sum(jul_net#149), partial_sum(aug_net#150), partial_sum(sep_net#151), partial_sum(oct_net#152), partial_sum(nov_net#153), partial_sum(dec_net#154)] - -(32) CometExchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130] -Functions [36]: [sum(jan_sales#131), sum(feb_sales#132), sum(mar_sales#133), sum(apr_sales#134), sum(may_sales#135), sum(jun_sales#136), sum(jul_sales#137), sum(aug_sales#138), sum(sep_sales#139), sum(oct_sales#140), sum(nov_sales#141), sum(dec_sales#142), sum((jan_sales#131 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#132 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#133 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#134 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#135 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#136 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#137 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#138 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#139 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#143), sum(feb_net#144), sum(mar_net#145), sum(apr_net#146), sum(may_net#147), sum(jun_net#148), sum(jul_net#149), sum(aug_net#150), sum(sep_net#151), sum(oct_net#152), sum(nov_net#153), sum(dec_net#154)] - -(34) CometTakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#13,w_country#14,ship_carriers#129,year#130,jan_sales#253,feb_sales#254,mar_sales#255,apr_sales#256,may_sales#257,jun_sales#258,jul_sales#259,aug_sales#260,sep_sales#261,oct_sales#262,nov_sales#263,dec_sales#264,jan_sales_per_sq_foot#265,feb_sales_per_sq_foot#266,mar_sales_per_sq_foot#267,apr_sales_per_sq_foot#268,may_sales_per_sq_foot#269,jun_sales_per_sq_foot#270,jul_sales_per_sq_foot#271,aug_sales_per_sq_foot#272,sep_sales_per_sq_foot#273,oct_sales_per_sq_foot#274,nov_sales_per_sq_foot#275,dec_sales_per_sq_foot#276,jan_net#277,feb_net#278,mar_net#279,apr_net#280,may_net#281,jun_net#282,jul_net#283,aug_net#284,sep_net#285,oct_net#286,nov_net#287,dec_net#288]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288], 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288] - -(35) ColumnarToRow [codegen id : 1] -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288] +(8) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] + +(12) CometBroadcastHashJoin +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight + +(13) CometProject +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(14) CometNativeScan parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#19, t_time#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) + +(16) CometProject +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] + +(17) CometBroadcastExchange +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] + +(18) CometBroadcastHashJoin +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight + +(19) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(20) CometNativeScan parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#22, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) + +(22) CometProject +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] + +(23) CometBroadcastExchange +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] + +(24) CometBroadcastHashJoin +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(26) CometHashAggregate +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(27) CometExchange +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometHashAggregate +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] +ReadSchema: struct + +(30) CometFilter +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(32) CometBroadcastHashJoin +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight + +(33) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(34) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] + +(35) CometBroadcastHashJoin +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight + +(36) CometProject +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(37) ReusedExchange [Reuses operator id: 17] +Output [1]: [t_time_sk#88] + +(38) CometBroadcastHashJoin +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight + +(39) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(40) ReusedExchange [Reuses operator id: 23] +Output [1]: [sm_ship_mode_sk#89] + +(41) CometBroadcastHashJoin +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight + +(42) CometProject +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(43) CometHashAggregate +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(44) CometExchange +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(45) CometHashAggregate +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(46) CometUnion +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] + +(47) CometHashAggregate +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] + +(48) CometExchange +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(49) CometHashAggregate +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] + +(50) CometTakeOrderedAndProject +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] + +(51) CometColumnarToRow [codegen id : 1] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/extended.txt new file mode 100644 index 0000000000..3dee6b4b4e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/extended.txt @@ -0,0 +1,64 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.time_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.ship_mode + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.ship_mode + +Comet accelerated 61 out of 61 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/simplified.txt index 3c072a2ea1..64ef752d91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] + CometHashAggregate [d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] @@ -18,20 +18,36 @@ WholeStageCodegen (1) CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [t_time_sk] #5 CometProject [t_time_sk] CometFilter [t_time_sk,t_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_time] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_time] CometBroadcastExchange [sm_ship_mode_sk] #6 CometProject [sm_ship_mode_sk] CometFilter [sm_ship_mode_sk,sm_carrier] - CometNativeScan: `spark_catalog`.`default`.`ship_mode` [sm_ship_mode_sk,sm_carrier] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] - ReusedExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 + CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/explain.txt index 99de5c768f..7012437ef9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/explain.txt @@ -1,48 +1,58 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * Sort (27) - +- Exchange (26) - +- WindowGroupLimit (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`item` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (34) ++- * Filter (33) + +- Window (32) + +- WindowGroupLimit (31) + +- * CometColumnarToRow (30) + +- CometSort (29) + +- CometColumnarExchange (28) + +- WindowGroupLimit (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.item (15) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] @@ -65,98 +75,115 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] +Right output [2]: [s_store_sk#11, s_store_id#13] Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(14) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] -(19) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(21) CometExpand +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] -(20) CometHashAggregate -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(22) CometHashAggregate +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(21) CometExchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#27, isEmpty#28] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(23) CometExchange +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#27, isEmpty#28] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(24) CometHashAggregate +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(23) CometSort -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29], [i_category#18 ASC NULLS FIRST, sumsales#29 DESC NULLS LAST] +(25) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] + +(26) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] -(24) ColumnarToRow [codegen id : 1] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] +(27) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Partial -(25) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18], [sumsales#29 DESC NULLS LAST], rank(sumsales#29), 100, Partial +(28) CometColumnarExchange +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(26) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(29) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] -(27) Sort [codegen id : 2] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#29 DESC NULLS LAST], false, 0 +(30) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] -(28) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18], [sumsales#29 DESC NULLS LAST], rank(sumsales#29), 100, Final +(31) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Final -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [rank(sumsales#29) windowspecdefinition(i_category#18, sumsales#29 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [i_category#18], [sumsales#29 DESC NULLS LAST] +(32) Window +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] -(30) Filter [codegen id : 3] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] -Condition : (rk#30 <= 100) +(33) Filter [codegen id : 3] +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#29 ASC NULLS FIRST, rk#30 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] +(34) TakeOrderedAndProject +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/extended.txt new file mode 100644 index 0000000000..36820cf87f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/extended.txt @@ -0,0 +1,36 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 27 out of 32 eligible operators (84%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/simplified.txt index 138c373993..dc17b92f12 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/simplified.txt @@ -5,33 +5,36 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (2) - Sort [i_category,sumsales] + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,spark_grouping_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] - CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [spark_grouping_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/explain.txt index 662ae98432..ca5c9ff3ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometProject (36) - +- CometBroadcastHashJoin (35) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) : :- CometHashAggregate (28) : : +- CometExchange (27) : : +- CometHashAggregate (26) @@ -17,39 +17,47 @@ : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : : : +- CometNativeScan parquet spark_catalog.default.store (9) : : : +- CometBroadcastExchange (18) : : : +- CometProject (17) : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- CometBroadcastExchange (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - +- ReusedExchange (34) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_dom#12] -Arguments: [d_date_sk#10, d_year#11, d_dom#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_dom#12] @@ -72,9 +80,12 @@ Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13, s_city#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#13, s_city#14] @@ -97,9 +108,12 @@ Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] @@ -122,9 +136,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(21) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct (22) CometFilter Input [2]: [ca_address_sk#18, ca_city#19] @@ -157,43 +174,50 @@ Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum# Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] -(29) CometNativeScan: `spark_catalog`.`default`.`customer` +(29) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (30) CometFilter Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] Condition : (isnotnull(c_customer_sk#23) AND isnotnull(c_current_addr_sk#24)) -(31) CometBroadcastExchange +(31) CometProject Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#23, c_current_addr_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#25, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#26, 30, true, false, true) AS c_last_name#28] + +(32) CometBroadcastExchange +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] -(32) CometBroadcastHashJoin -Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, extended_price#28, list_price#29, extended_tax#30] -Right output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +(33) CometBroadcastHashJoin +Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32] +Right output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] Arguments: [ss_customer_sk#1], [c_customer_sk#23], Inner, BuildRight -(33) CometProject -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Arguments: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26], [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +(34) CometProject +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] -(34) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#31, ca_city#32] +(35) ReusedExchange [Reuses operator id: 23] +Output [2]: [ca_address_sk#33, ca_city#34] -(35) CometBroadcastHashJoin -Left output [8]: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Right output [2]: [ca_address_sk#31, ca_city#32] -Arguments: [c_current_addr_sk#24], [ca_address_sk#31], Inner, NOT (ca_city#32 = bought_city#27), BuildRight +(36) CometBroadcastHashJoin +Left output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Right output [2]: [ca_address_sk#33, ca_city#34] +Arguments: [c_current_addr_sk#24], [ca_address_sk#33], Inner, NOT (ca_city#34 = bought_city#29), BuildRight -(36) CometProject -Input [10]: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26, ca_address_sk#31, ca_city#32] -Arguments: [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29], [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] +(37) CometProject +Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28, ca_address_sk#33, ca_city#34] +Arguments: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -(37) CometTakeOrderedAndProject -Input [8]: [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#26 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#26,c_first_name#25,ca_city#32,bought_city#27,ss_ticket_number#5,extended_price#28,extended_tax#30,list_price#29]), [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29], 100, [c_last_name#26 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] +(38) CometTakeOrderedAndProject +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#28 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#28,c_first_name#27,ca_city#34,bought_city#29,ss_ticket_number#5,extended_price#30,extended_tax#32,list_price#31]), [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], 100, 0, [c_last_name#28 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -(38) ColumnarToRow [codegen id : 1] -Input [8]: [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] +(39) CometColumnarToRow [codegen id : 1] +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/extended.txt new file mode 100644 index 0000000000..2cc4ab6ef2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 40 out of 40 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/simplified.txt index 9fbeb0195c..fe430a0ce4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] CometBroadcastHashJoin [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] CometProject [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,ss_addr_sk,ca_city,sum,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax))] + CometHashAggregate [ss_addr_sk,sum,sum,sum] [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,ca_city,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax))] CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometHashAggregate [ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] @@ -18,23 +18,24 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_city] CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/explain.txt index 709357a617..2007f6644d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/explain.txt @@ -1,60 +1,76 @@ == Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin LeftAnti BuildRight (21) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (46) ++- * HashAggregate (45) + +- * CometColumnarToRow (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (39) + +- * CometColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (35) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6, d_year#7, d_moy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] @@ -86,12 +102,15 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#11] @@ -105,7 +124,7 @@ Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#9] (18) BroadcastExchange @@ -118,92 +137,128 @@ Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#12] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] +ReadSchema: struct -(21) BroadcastHashJoin [codegen id : 5] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#14] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [cs_ship_customer_sk#12], [cs_ship_customer_sk#12] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#12] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] Right keys [1]: [cs_ship_customer_sk#12] Join type: LeftAnti Join condition: None -(22) Project [codegen id : 5] +(27) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13, ca_state#14] +(28) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(24) CometFilter -Input [2]: [ca_address_sk#13, ca_state#14] -Condition : (ca_state#14 IN (KY,GA,NM) AND isnotnull(ca_address_sk#13)) +(29) CometFilter +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) -(25) CometProject -Input [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13], [ca_address_sk#13] +(30) CometProject +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] -(26) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#13] +(31) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#15] -(27) BroadcastExchange -Input [1]: [ca_address_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(32) BroadcastExchange +Input [1]: [ca_address_sk#15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(28) BroadcastHashJoin [codegen id : 5] +(33) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#13] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(29) Project [codegen id : 5] +(34) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#13] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(35) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(31) CometFilter -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Condition : isnotnull(cd_demo_sk#15) +(36) CometFilter +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Condition : isnotnull(cd_demo_sk#17) -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(37) CometProject +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Arguments: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26], [cd_demo_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#18, 1, true, false, true) AS cd_gender#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#25, cd_purchase_estimate#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#22, 10, true, false, true) AS cd_credit_rating#26] -(33) BroadcastExchange -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(38) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] -(34) BroadcastHashJoin [codegen id : 5] +(39) BroadcastExchange +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#15] +Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: None -(35) Project [codegen id : 5] -Output [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(41) Project [codegen id : 5] +Output [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] -(36) HashAggregate [codegen id : 5] -Input [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(42) HashAggregate [codegen id : 5] +Input [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#21] -Results [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] +Aggregate Attributes [1]: [count#27] +Results [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] + +(43) CometColumnarExchange +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(37) Exchange -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Arguments: hashpartitioning(cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(44) CometColumnarToRow [codegen id : 6] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] -(38) HashAggregate [codegen id : 6] -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(45) HashAggregate [codegen id : 6] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#23] -Results [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, count(1)#23 AS cnt1#24, cd_purchase_estimate#19, count(1)#23 AS cnt2#25, cd_credit_rating#20, count(1)#23 AS cnt3#26] +Aggregate Attributes [1]: [count(1)#29] +Results [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#29 AS cnt1#30, cd_purchase_estimate#21, count(1)#29 AS cnt2#31, cd_credit_rating#26, count(1)#29 AS cnt3#32] -(39) TakeOrderedAndProject -Input [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] -Arguments: 100, [cd_gender#16 ASC NULLS FIRST, cd_marital_status#17 ASC NULLS FIRST, cd_education_status#18 ASC NULLS FIRST, cd_purchase_estimate#19 ASC NULLS FIRST, cd_credit_rating#20 ASC NULLS FIRST], [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] +(46) TakeOrderedAndProject +Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] +Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#26 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/extended.txt new file mode 100644 index 0000000000..0fa566e7ff --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/extended.txt @@ -0,0 +1,54 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 46 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/simplified.txt index f8b77163ae..69d8e49a63 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/simplified.txt @@ -1,53 +1,62 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/explain.txt index 135833f1ca..5f35ffe250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.promotion (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -64,9 +72,12 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#13, d_year#14] @@ -89,70 +100,80 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] +Right output [2]: [i_item_sk#15, i_item_id#17] Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [ss_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(26) CometHashAggregate -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] + +(21) CometNativeScan parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) + +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] + +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight + +(26) CometProject +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] + +(27) CometHashAggregate +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometExchange -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) CometHashAggregate -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -(29) CometTakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#16 ASC NULLS FIRST], output=[i_item_id#16,agg1#28,agg2#29,agg3#30,agg4#31]), [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31], 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] -(30) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/extended.txt new file mode 100644 index 0000000000..4848e7820f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/simplified.txt index a0d42bf3ba..0867ca71bb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] @@ -14,19 +14,20 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/explain.txt index a2e5e6afea..6cb4a45b19 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/explain.txt @@ -1,62 +1,73 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Expand (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- BroadcastExchange (34) - +- * BroadcastHashJoin LeftSemi BuildRight (33) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (10) - +- BroadcastExchange (32) - +- * Project (31) - +- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * ColumnarToRow (27) - +- CometSort (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- ReusedExchange (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (49) ++- * Project (48) + +- Window (47) + +- * CometColumnarToRow (46) + +- CometSort (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometColumnarExchange (41) + +- * HashAggregate (40) + +- * Expand (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (36) + +- * Project (35) + +- * BroadcastHashJoin LeftSemi BuildRight (34) + :- * CometColumnarToRow (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.store (10) + +- BroadcastExchange (33) + +- * Project (32) + +- * Filter (31) + +- Window (30) + +- WindowGroupLimit (29) + +- * CometColumnarToRow (28) + +- CometSort (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometFilter (14) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + +- ReusedExchange (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -79,161 +90,185 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) ColumnarToRow [codegen id : 4] +(9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometNativeScan: `spark_catalog`.`default`.`store` +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: [s_store_sk#6, s_county#7, s_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] Condition : isnotnull(s_store_sk#6) -(12) ColumnarToRow [codegen id : 3] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Arguments: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#12, s_state#13] Condition : isnotnull(s_store_sk#12) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] -(18) CometBroadcastHashJoin +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] +Right output [2]: [s_store_sk#12, s_state#14] Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] -(23) CometHashAggregate -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -(24) CometExchange -Input [2]: [s_state#13, sum#15] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) CometHashAggregate -Input [2]: [s_state#13, sum#15] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -(26) CometSort -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13, _w0#16, s_state#13], [s_state#13 ASC NULLS FIRST, _w0#16 DESC NULLS LAST] +(27) CometSort +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(27) ColumnarToRow [codegen id : 1] -Input [3]: [s_state#13, _w0#16, s_state#13] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#14, _w0#17, s_state#14] -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13], [_w0#16 DESC NULLS LAST], rank(_w0#16), 5, Final +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14], [_w0#17 DESC NULLS LAST], rank(_w0#17), 5, Final -(29) Window -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [rank(_w0#16) windowspecdefinition(s_state#13, _w0#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#17], [s_state#13], [_w0#16 DESC NULLS LAST] +(30) Window +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] -(30) Filter [codegen id : 2] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] -Condition : (ranking#17 <= 5) +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) -(31) Project [codegen id : 2] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] -(32) BroadcastExchange -Input [1]: [s_state#13] +(33) BroadcastExchange +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] -(33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None -(34) BroadcastExchange +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 4] +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(36) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_state#19, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] -(37) Expand [codegen id : 4] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#18, s_county#19, spark_grouping_id#20] +(39) Expand [codegen id : 4] +Input [3]: [ss_net_profit#2, s_state#19, s_county#7] +Arguments: [[ss_net_profit#2, s_state#19, s_county#7, 0], [ss_net_profit#2, s_state#19, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] -(38) HashAggregate [codegen id : 4] -Input [4]: [ss_net_profit#2, s_state#18, s_county#19, spark_grouping_id#20] -Keys [3]: [s_state#18, s_county#19, spark_grouping_id#20] +(40) HashAggregate [codegen id : 4] +Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] +Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] +Aggregate Attributes [1]: [sum#23] +Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -(39) Exchange -Input [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] -Arguments: hashpartitioning(s_state#18, s_county#19, spark_grouping_id#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(41) CometColumnarExchange +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 5] -Input [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] -Keys [3]: [s_state#18, s_county#19, spark_grouping_id#20] +(42) CometColumnarToRow [codegen id : 5] +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] + +(43) HashAggregate [codegen id : 5] +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS total_sum#24, s_state#18, s_county#19, (cast((shiftright(spark_grouping_id#20, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint)) AS lochierarchy#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS _w0#26, (cast((shiftright(spark_grouping_id#20, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint)) AS _w1#27, CASE WHEN (cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint) = 0) THEN s_state#18 END AS _w2#28] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] + +(44) CometColumnarExchange +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(41) Exchange -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: hashpartitioning(_w1#27, _w2#28, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(45) CometSort +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30], [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST] -(42) Sort [codegen id : 6] -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: [_w1#27 ASC NULLS FIRST, _w2#28 ASC NULLS FIRST, _w0#26 DESC NULLS LAST], false, 0 +(46) CometColumnarToRow [codegen id : 6] +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -(43) Window -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: [rank(_w0#26) windowspecdefinition(_w1#27, _w2#28, _w0#26 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#29], [_w1#27, _w2#28], [_w0#26 DESC NULLS LAST] +(47) Window +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] -(44) Project [codegen id : 7] -Output [5]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] -Input [8]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28, rank_within_parent#29] +(48) Project [codegen id : 7] +Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] -(45) TakeOrderedAndProject -Input [5]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] -Arguments: 100, [lochierarchy#25 DESC NULLS LAST, CASE WHEN (lochierarchy#25 = 0) THEN s_state#18 END ASC NULLS FIRST, rank_within_parent#29 ASC NULLS FIRST], [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] +(49) TakeOrderedAndProject +Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/extended.txt new file mode 100644 index 0000000000..3badd993dd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/extended.txt @@ -0,0 +1,54 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 31 out of 47 eligible operators (65%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/simplified.txt index 963f843456..2e1b843f68 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/simplified.txt @@ -4,58 +4,62 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (6) - Sort [_w1,_w2,_w0] + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #6 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #7 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - ReusedExchange [d_date_sk] #3 + CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #6 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #7 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/explain.txt index 6f27297066..b0d17c994e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -* ColumnarToRow (37) +* CometColumnarToRow (37) +- CometSort (36) - +- CometColumnarExchange (35) + +- CometExchange (35) +- CometHashAggregate (34) +- CometExchange (33) +- CometHashAggregate (32) @@ -12,35 +12,38 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : +- CometNativeScan parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) : : :- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (5) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (5) : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (7) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (13) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (13) : : +- ReusedExchange (15) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (19) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (18) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (20) +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (26) + +- CometNativeScan parquet spark_catalog.default.time_dim (26) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] @@ -48,146 +51,164 @@ Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull( (3) CometProject Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3], [i_item_sk#1, i_brand_id#2, i_brand#3] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#3, 50, true, false, true) AS i_brand#5] (4) CometBroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3] +Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(5) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Arguments: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +(5) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] +ReadSchema: struct (6) CometFilter -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) +Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) -(7) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] +(7) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (8) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) (9) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (10) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (11) CometBroadcastHashJoin -Left output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (12) CometProject -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] -Arguments: [ext_price#12, sold_item_sk#13, time_sk#14], [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(13) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] +ReadSchema: struct (14) CometFilter -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) (15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#20] (16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (17) CometProject -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [ext_price#20, sold_item_sk#21, time_sk#22], [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] -(18) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Arguments: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(18) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] +ReadSchema: struct (19) CometFilter -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) (20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#27] +Output [1]: [d_date_sk#28] (21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (22) CometProject -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] -Arguments: [ext_price#28, sold_item_sk#29, time_sk#30], [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] (23) CometUnion -Child 0 Input [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Child 1 Input [3]: [ext_price#20, sold_item_sk#21, time_sk#22] -Child 2 Input [3]: [ext_price#28, sold_item_sk#29, time_sk#30] +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] (24) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Right output [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_item_sk#1], [sold_item_sk#13], Inner, BuildLeft +Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft (25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14], [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] -(26) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] +(26) CometNativeScan parquet spark_catalog.default.time_dim +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_time_sk)] +ReadSchema: struct (27) CometFilter -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#32)) (28) CometProject -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33], [t_time_sk#31, t_hour#32, t_minute#33] +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] (29) CometBroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] (30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Right output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [time_sk#14], [t_time_sk#31], Inner, BuildRight +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight (31) CometProject -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33], [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] (32) CometHashAggregate -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] (33) CometExchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#35] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (34) CometHashAggregate -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#35] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] -(35) CometColumnarExchange -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] -Arguments: rangepartitioning(ext_price#38 DESC NULLS LAST, brand_id#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(35) CometExchange +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: rangepartitioning(ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (36) CometSort -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] -Arguments: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38], [ext_price#38 DESC NULLS LAST, brand_id#36 ASC NULLS FIRST] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39], [ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST] -(37) ColumnarToRow [codegen id : 1] -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] +(37) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/extended.txt new file mode 100644 index 0000000000..4bd9ef86c7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometUnion + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.time_dim + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/simplified.txt index 98df4c26e6..be00094510 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/simplified.txt @@ -1,39 +1,39 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [brand_id,brand,t_hour,t_minute,ext_price] - CometColumnarExchange [ext_price,brand_id] #1 - CometHashAggregate [brand_id,brand,t_hour,t_minute,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ext_price))] + CometExchange [ext_price,brand_id] #1 + CometHashAggregate [sum] [brand_id,brand,t_hour,t_minute,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ext_price))] CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 - CometHashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum,ext_price] + CometHashAggregate [ext_price] [i_brand,i_brand_id,t_hour,t_minute,sum] CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] CometProject [i_brand_id,i_brand,ext_price,time_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] CometUnion [ext_price,sold_item_sk,time_sk] CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #4 CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #4 CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 CometProject [t_time_sk,t_hour,t_minute] CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute,t_meal_time] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/explain.txt index 2ef4c4fa34..5ffc831b1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/explain.txt @@ -1,79 +1,89 @@ == Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometHashAggregate (60) - +- CometExchange (59) - +- CometHashAggregate (58) - +- CometProject (57) - +- CometSortMergeJoin (56) - :- CometSort (50) - : +- CometExchange (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometBroadcastHashJoin (37) - : : : :- CometProject (33) - : : : : +- CometBroadcastHashJoin (32) - : : : : :- CometProject (27) - : : : : : +- CometBroadcastHashJoin (26) - : : : : : :- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) : : : : : : : :- CometProject (12) : : : : : : : : +- CometBroadcastHashJoin (11) : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (3) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : : : : : : : +- ReusedExchange (13) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (16) - : : : : : +- CometBroadcastExchange (25) - : : : : : +- CometProject (24) - : : : : : +- CometFilter (23) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (22) - : : : : +- CometBroadcastExchange (31) - : : : : +- CometProject (30) - : : : : +- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (28) - : : : +- CometBroadcastExchange (36) - : : : +- CometFilter (35) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (44) - +- CometSort (55) - +- CometExchange (54) - +- CometProject (53) - +- CometFilter (52) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (51) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`inventory` +(3) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#12)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] @@ -92,9 +102,12 @@ Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] @@ -113,211 +126,244 @@ Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -(13) ReusedExchange [Reuses operator id: 10] +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_desc#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) -(14) CometBroadcastHashJoin +(15) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] + +(16) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] Right output [2]: [i_item_sk#15, i_item_desc#16] Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight -(15) CometProject +(17) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(16) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17, cd_marital_status#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(17) CometFilter +(19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#17)) -(18) CometProject +(20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] Arguments: [cd_demo_sk#17], [cd_demo_sk#17] -(19) CometBroadcastExchange +(21) CometBroadcastExchange Input [1]: [cd_demo_sk#17] Arguments: [cd_demo_sk#17] -(20) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [1]: [cd_demo_sk#17] Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight -(21) CometProject +(23) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(22) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19, hd_buy_potential#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk)] +ReadSchema: struct -(23) CometFilter +(25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#19)) -(24) CometProject +(26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] Arguments: [hd_demo_sk#19], [hd_demo_sk#19] -(25) CometBroadcastExchange +(27) CometBroadcastExchange Input [1]: [hd_demo_sk#19] Arguments: [hd_demo_sk#19] -(26) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [1]: [hd_demo_sk#19] Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight -(27) CometProject +(29) CometProject Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(28) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] +ReadSchema: struct -(29) CometFilter +(31) CometFilter Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) -(30) CometProject +(32) CometProject Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] -(31) CometBroadcastExchange +(33) CometBroadcastExchange Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] -(32) CometBroadcastHashJoin +(34) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight -(33) CometProject +(35) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(35) CometFilter +(37) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(36) CometBroadcastExchange +(38) CometBroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] Arguments: [d_date_sk#25, d_week_seq#26] -(37) CometBroadcastHashJoin +(39) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] Right output [2]: [d_date_sk#25, d_week_seq#26] Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight -(38) CometProject +(40) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] +ReadSchema: struct -(40) CometFilter +(42) CometFilter Input [2]: [d_date_sk#27, d_date#28] Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(41) CometBroadcastExchange +(43) CometBroadcastExchange Input [2]: [d_date_sk#27, d_date#28] Arguments: [d_date_sk#27, d_date#28] -(42) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] Right output [2]: [d_date_sk#27, d_date#28] Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight -(43) CometProject +(45) CometProject Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(44) CometNativeScan: `spark_catalog`.`default`.`promotion` +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct -(45) CometFilter +(47) CometFilter Input [1]: [p_promo_sk#29] Condition : isnotnull(p_promo_sk#29) -(46) CometBroadcastExchange +(48) CometBroadcastExchange Input [1]: [p_promo_sk#29] Arguments: [p_promo_sk#29] -(47) CometBroadcastHashJoin +(49) CometBroadcastHashJoin Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Right output [1]: [p_promo_sk#29] Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight -(48) CometProject +(50) CometProject Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(49) CometExchange +(51) CometExchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(50) CometSort +(52) CometSort Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(51) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct -(52) CometFilter +(54) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(53) CometProject +(55) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] -(54) CometExchange +(56) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(55) CometSort +(57) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] -(56) CometSortMergeJoin +(58) CometSortMergeJoin Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Right output [2]: [cr_item_sk#30, cr_order_number#31] Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter -(57) CometProject +(59) CometProject Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(58) CometHashAggregate +(60) CometHashAggregate Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] -(59) CometExchange +(61) CometExchange Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(60) CometHashAggregate +(62) CometHashAggregate Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] -(61) CometTakeOrderedAndProject +(63) CometTakeOrderedAndProject Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -(62) ColumnarToRow [codegen id : 1] +(64) CometColumnarToRow [codegen id : 1] Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/extended.txt new file mode 100644 index 0000000000..3bf43ef8c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/extended.txt @@ -0,0 +1,66 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_returns + +Comet accelerated 63 out of 63 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/simplified.txt index e70defef63..073bd57a8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] CometProject [w_warehouse_name,i_item_desc,d_week_seq] @@ -28,37 +28,39 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [i_item_sk,i_item_desc] #4 - CometBroadcastExchange [cd_demo_sk] #5 + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #6 + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #7 + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #8 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #9 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #10 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #11 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/explain.txt index 13f2c1319f..f4fcae1584 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,35 +15,43 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] @@ -66,9 +74,12 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#9, s_county#10] @@ -91,13 +102,16 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -134,35 +148,42 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) -(25) CometNativeScan: `spark_catalog`.`default`.`customer` +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (26) CometFilter Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Condition : isnotnull(c_customer_sk#17) -(27) CometBroadcastExchange +(27) CometProject Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(28) CometBroadcastHashJoin +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/simplified.txt index 35a931bfdf..145db18334 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [cnt] #1 + CometExchange [cnt] #1 CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] @@ -16,19 +16,20 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/explain.txt index f2a45c5e59..571fdd617c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/explain.txt @@ -1,232 +1,383 @@ == Physical Plan == -* ColumnarToRow (44) -+- CometTakeOrderedAndProject (43) - +- CometProject (42) - +- CometBroadcastHashJoin (41) - :- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - +- CometBroadcastExchange (40) - +- CometHashAggregate (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(13) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] - -(14) CometExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] - -(16) CometFilter -Input [2]: [customer_id#11, year_total#12] -Condition : (isnotnull(year_total#12) AND (year_total#12 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Arguments: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] - -(18) CometFilter -Input [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Condition : (isnotnull(c_customer_sk#13) AND isnotnull(c_customer_id#14)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] - -(20) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Right output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_sk#13], [ss_customer_sk#17], Inner, BuildRight - -(21) CometProject -Input [7]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16, ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(23) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : (((isnotnull(d_year#21) AND (d_year#21 = 2002)) AND d_year#21 IN (2001,2002)) AND isnotnull(d_date_sk#20)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(25) CometBroadcastHashJoin -Left output [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight - -(26) CometProject -Input [7]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] - -(27) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#18))] - -(28) CometExchange -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Arguments: hashpartitioning(c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) + +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) + +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] + +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight + +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] + +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) + +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] + +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] + +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight + +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] + +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight + +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] (29) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [sum(UnscaledValue(ss_net_paid#18))] - -(30) CometBroadcastExchange -Input [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#11, year_total#12] -Right output [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#11], [customer_id#23], Inner, BuildRight - -(32) ReusedExchange [Reuses operator id: 14] -Output [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] - -(33) CometHashAggregate -Input [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] -Keys [4]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30] -Functions [1]: [sum(UnscaledValue(ws_net_paid#32))] - -(34) CometFilter -Input [2]: [customer_id#33, year_total#34] -Condition : (isnotnull(year_total#34) AND (year_total#34 > 0.00)) - -(35) CometBroadcastExchange -Input [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#33, year_total#34] - -(36) CometBroadcastHashJoin -Left output [6]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Right output [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#11], [customer_id#33], Inner, BuildRight - -(37) CometProject -Input [8]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, customer_id#33, year_total#34] -Arguments: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34], [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] - -(38) ReusedExchange [Reuses operator id: 28] -Output [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] - -(39) CometHashAggregate -Input [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] -Keys [4]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(40) CometBroadcastExchange -Input [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#41, year_total#42] - -(41) CometBroadcastHashJoin -Left output [7]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] -Right output [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#11], [customer_id#41], Inner, (CASE WHEN (year_total#34 > 0.00) THEN (year_total#42 / year_total#34) END > CASE WHEN (year_total#12 > 0.00) THEN (year_total#26 / year_total#12) END), BuildRight - -(42) CometProject -Input [9]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34, customer_id#41, year_total#42] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(43) CometTakeOrderedAndProject -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#23 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST], output=[customer_id#23,customer_first_name#24,customer_last_name#25]), [customer_id#23, customer_first_name#24, customer_last_name#25], 100, [customer_id#23 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(44) ColumnarToRow [codegen id : 1] -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight + +(34) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) + +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] + +(37) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) + +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] + +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight + +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] + +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] + +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight + +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] + +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] + +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] + +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] + +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight + +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/extended.txt new file mode 100644 index 0000000000..bcd6c3e73d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 75 out of 75 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/simplified.txt index ea904874f9..95b5b340e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometProject [customer_id,customer_first_name,customer_last_name] @@ -8,39 +8,63 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #1 - CometBroadcastExchange [customer_id,year_total] #8 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #5 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/explain.txt index 093bb56034..f542de162d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/explain.txt @@ -1,18 +1,18 @@ == Physical Plan == -* ColumnarToRow (86) -+- CometTakeOrderedAndProject (85) - +- CometProject (84) - +- CometSortMergeJoin (83) - :- CometSort (44) - : +- CometExchange (43) - : +- CometFilter (42) - : +- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometUnion (35) +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometProject (122) + +- CometSortMergeJoin (121) + :- CometSort (66) + : +- CometExchange (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) : :- CometProject (22) : : +- CometSortMergeJoin (21) : : :- CometSort (15) @@ -22,86 +22,131 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : :- CometProject (28) - : : +- CometSortMergeJoin (27) - : : :- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometSort (26) - : : +- ReusedExchange (25) - : +- CometProject (34) - : +- CometSortMergeJoin (33) - : :- CometSort (30) - : : +- ReusedExchange (29) - : +- CometSort (32) - : +- ReusedExchange (31) - +- CometSort (82) - +- CometExchange (81) - +- CometFilter (80) - +- CometHashAggregate (79) - +- CometExchange (78) - +- CometHashAggregate (77) - +- CometHashAggregate (76) - +- CometExchange (75) - +- CometHashAggregate (74) - +- CometUnion (73) - :- CometProject (60) - : +- CometSortMergeJoin (59) - : :- CometSort (56) - : : +- CometExchange (55) - : : +- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometProject (49) - : : : +- CometBroadcastHashJoin (48) - : : : :- CometFilter (46) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (45) - : : : +- ReusedExchange (47) - : : +- CometBroadcastExchange (52) - : : +- CometFilter (51) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (50) - : +- CometSort (58) - : +- ReusedExchange (57) - :- CometProject (66) - : +- CometSortMergeJoin (65) - : :- CometSort (62) - : : +- ReusedExchange (61) - : +- CometSort (64) - : +- ReusedExchange (63) - +- CometProject (72) - +- CometSortMergeJoin (71) - :- CometSort (68) - : +- ReusedExchange (67) - +- CometSort (70) - +- ReusedExchange (69) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) + +- CometSort (120) + +- CometExchange (119) + +- CometFilter (118) + +- CometHashAggregate (117) + +- CometExchange (116) + +- CometHashAggregate (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometProject (82) + : +- CometSortMergeJoin (81) + : :- CometSort (78) + : : +- CometExchange (77) + : : +- CometProject (76) + : : +- CometBroadcastHashJoin (75) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometFilter (68) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (67) + : : : +- ReusedExchange (69) + : : +- CometBroadcastExchange (74) + : : +- CometFilter (73) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) + : +- CometSort (80) + : +- ReusedExchange (79) + :- CometProject (96) + : +- CometSortMergeJoin (95) + : :- CometSort (92) + : : +- CometExchange (91) + : : +- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometFilter (84) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (83) + : : : +- ReusedExchange (85) + : : +- ReusedExchange (88) + : +- CometSort (94) + : +- ReusedExchange (93) + +- CometProject (110) + +- CometSortMergeJoin (109) + :- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometBroadcastHashJoin (103) + : :- CometProject (101) + : : +- CometBroadcastHashJoin (100) + : : :- CometFilter (98) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (97) + : : +- ReusedExchange (99) + : +- ReusedExchange (102) + +- CometSort (108) + +- ReusedExchange (107) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] @@ -120,9 +165,12 @@ Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#12, d_year#13] @@ -149,9 +197,12 @@ Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREM Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct (17) CometFilter Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] @@ -178,266 +229,453 @@ Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14 Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(23) ReusedExchange [Reuses operator id: 14] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] +(23) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct -(24) CometSort -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] +(24) CometFilter +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) -(25) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] +(25) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(26) CometSort -Input [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33], [sr_ticket_number#31 ASC NULLS FIRST, sr_item_sk#30 ASC NULLS FIRST] +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight -(27) CometSortMergeJoin -Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Right output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#31, sr_item_sk#30], LeftOuter +(27) CometProject +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(28) CometProject -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29, sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35], [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, (ss_quantity#23 - coalesce(sr_return_quantity#32, 0)) AS sales_cnt#34, (ss_ext_sales_price#24 - coalesce(sr_return_amt#33, 0.00)) AS sales_amt#35] +(28) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#31, d_year#32] -(29) ReusedExchange [Reuses operator id: 14] -Output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight -(30) CometSort -Input [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Arguments: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44], [ws_order_number#37 ASC NULLS FIRST, ws_item_sk#36 ASC NULLS FIRST] +(30) CometProject +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -(31) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] +(31) CometExchange +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48], [wr_order_number#46 ASC NULLS FIRST, wr_item_sk#45 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] + +(33) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) + +(35) CometProject +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] + +(36) CometExchange +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(37) CometSort +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] + +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter + +(39) CometProject +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] + +(40) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(41) CometFilter +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) + +(42) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight + +(44) CometProject +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#50, d_year#51] + +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight + +(47) CometProject +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] + +(48) CometExchange +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(49) CometSort +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] + +(50) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) + +(52) CometProject +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] + +(53) CometExchange +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(33) CometSortMergeJoin -Left output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Right output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [ws_order_number#37, ws_item_sk#36], [wr_order_number#46, wr_item_sk#45], LeftOuter +(54) CometSort +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] -(34) CometProject -Input [13]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44, wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50], [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, (ws_quantity#38 - coalesce(wr_return_quantity#47, 0)) AS sales_cnt#49, (ws_ext_sales_price#39 - coalesce(wr_return_amt#48, 0.00)) AS sales_amt#50] +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter -(35) CometUnion +(56) CometProject +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] + +(57) CometUnion Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] -(36) CometHashAggregate +(58) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -(37) CometExchange +(59) CometExchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(38) CometHashAggregate +(60) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -(39) CometHashAggregate +(61) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -(40) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(41) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] +(63) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -(42) CometFilter -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Condition : isnotnull(sales_cnt#53) - -(43) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(44) CometSort -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] - -(45) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] - -(46) CometFilter -Input [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Condition : isnotnull(cs_item_sk#55) - -(47) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(48) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Right output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55], [i_item_sk#60], Inner, BuildRight - -(49) CometProject -Input [10]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(50) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(51) CometFilter -Input [2]: [d_date_sk#65, d_year#66] -Condition : ((isnotnull(d_year#66) AND (d_year#66 = 2001)) AND isnotnull(d_date_sk#65)) - -(52) CometBroadcastExchange -Input [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(53) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [cs_sold_date_sk#59], [d_date_sk#65], Inner, BuildRight - -(54) CometProject -Input [11]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_date_sk#65, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] - -(55) CometExchange -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(56) CometSort -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_order_number#56 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST] - -(57) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] - -(58) CometSort -Input [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70], [cr_order_number#68 ASC NULLS FIRST, cr_item_sk#67 ASC NULLS FIRST] - -(59) CometSortMergeJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Right output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cs_order_number#56, cs_item_sk#55], [cr_order_number#68, cr_item_sk#67], LeftOuter - -(60) CometProject -Input [13]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66, cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20], [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, (cs_quantity#57 - coalesce(cr_return_quantity#69, 0)) AS sales_cnt#19, (cs_ext_sales_price#58 - coalesce(cr_return_amount#70, 0.00)) AS sales_amt#20] - -(61) ReusedExchange [Reuses operator id: 55] -Output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] - -(62) CometSort -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79], [ss_ticket_number#72 ASC NULLS FIRST, ss_item_sk#71 ASC NULLS FIRST] - -(63) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] - -(64) CometSort -Input [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83], [sr_ticket_number#81 ASC NULLS FIRST, sr_item_sk#80 ASC NULLS FIRST] - -(65) CometSortMergeJoin -Left output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Right output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [ss_ticket_number#72, ss_item_sk#71], [sr_ticket_number#81, sr_item_sk#80], LeftOuter - -(66) CometProject -Input [13]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79, sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35], [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, (ss_quantity#73 - coalesce(sr_return_quantity#82, 0)) AS sales_cnt#34, (ss_ext_sales_price#74 - coalesce(sr_return_amt#83, 0.00)) AS sales_amt#35] - -(67) ReusedExchange [Reuses operator id: 55] -Output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] - -(68) CometSort -Input [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Arguments: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92], [ws_order_number#85 ASC NULLS FIRST, ws_item_sk#84 ASC NULLS FIRST] - -(69) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] - -(70) CometSort -Input [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96], [wr_order_number#94 ASC NULLS FIRST, wr_item_sk#93 ASC NULLS FIRST] - -(71) CometSortMergeJoin -Left output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Right output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [ws_order_number#85, ws_item_sk#84], [wr_order_number#94, wr_item_sk#93], LeftOuter - -(72) CometProject -Input [13]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92, wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50], [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, (ws_quantity#86 - coalesce(wr_return_quantity#95, 0)) AS sales_cnt#49, (ws_ext_sales_price#87 - coalesce(wr_return_amt#96, 0.00)) AS sales_amt#50] - -(73) CometUnion -Child 0 Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50] - -(74) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] +(64) CometFilter +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) + +(65) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(66) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] + +(67) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(68) CometFilter +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) + +(69) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] + +(70) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight + +(71) CometProject +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] + +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#73, d_year#74] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(73) CometFilter +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) + +(74) CometBroadcastExchange +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] + +(75) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight + +(76) CometProject +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] + +(77) CometExchange +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(78) CometSort +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] + +(79) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] + +(80) CometSort +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] + +(81) CometSortMergeJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter + +(82) CometProject +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] + +(83) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(84) CometFilter +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) + +(85) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] + +(86) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight + +(87) CometProject +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] + +(88) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#89, d_year#90] + +(89) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight + +(90) CometProject +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] + +(91) CometExchange +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(92) CometSort +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] + +(93) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] + +(94) CometSort +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] + +(95) CometSortMergeJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter + +(96) CometProject +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] + +(97) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(98) CometFilter +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) + +(99) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] + +(100) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight + +(101) CometProject +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] + +(102) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#105, d_year#106] + +(103) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight + +(104) CometProject +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] + +(105) CometExchange +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(106) CometSort +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] + +(107) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] + +(108) CometSort +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] + +(109) CometSortMergeJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter + +(110) CometProject +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] + +(111) CometUnion +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] + +(112) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -(75) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(113) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(76) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] +(114) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -(77) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] +(115) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -(78) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(116) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(79) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] +(117) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -(80) CometFilter -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Condition : isnotnull(sales_cnt#98) +(118) CometFilter +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) -(81) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +(119) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(82) CometSort -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99], [i_brand_id#61 ASC NULLS FIRST, i_class_id#62 ASC NULLS FIRST, i_category_id#63 ASC NULLS FIRST, i_manufact_id#64 ASC NULLS FIRST] +(120) CometSort +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] -(83) CometSortMergeJoin -Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Right output [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], Inner, ((cast(sales_cnt#53 as decimal(17,2)) / cast(sales_cnt#98 as decimal(17,2))) < 0.90000000000000000000) +(121) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) -(84) CometProject -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54, d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], [d_year#66 AS prev_year#100, d_year#13 AS year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#98 AS prev_yr_cnt#102, sales_cnt#53 AS curr_yr_cnt#103, (sales_cnt#53 - sales_cnt#98) AS sales_cnt_diff#104, (sales_amt#54 - sales_amt#99) AS sales_amt_diff#105] +(122) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] -(85) CometTakeOrderedAndProject -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#104 ASC NULLS FIRST], output=[prev_year#100,year#101,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#102,curr_yr_cnt#103,sales_cnt_diff#104,sales_amt_diff#105]), [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], 100, [sales_cnt_diff#104 ASC NULLS FIRST], [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] +(123) CometTakeOrderedAndProject +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] -(86) ColumnarToRow [codegen id : 1] -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] +(124) CometColumnarToRow [codegen id : 1] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/extended.txt new file mode 100644 index 0000000000..84620b41ae --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/extended.txt @@ -0,0 +1,158 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_returns + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_returns + +- CometSort + +- CometExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_returns + +Comet accelerated 155 out of 155 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/simplified.txt index 6503d71685..588e8943c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] @@ -7,9 +7,9 @@ WholeStageCodegen (1) CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -23,66 +23,104 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #8 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #9 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #10 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #11 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometBroadcastExchange [d_date_sk,d_year] #12 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/explain.txt index 635e8158a2..f04f5a8a5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/explain.txt @@ -1,177 +1,200 @@ == Physical Plan == -* ColumnarToRow (34) -+- CometTakeOrderedAndProject (33) - +- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometUnion (29) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (35) ++- CometTakeOrderedAndProject (34) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometUnion (30) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - :- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : +- ReusedExchange (15) - : +- ReusedExchange (18) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (21) - : +- ReusedExchange (23) - +- ReusedExchange (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + :- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (14) + : : +- ReusedExchange (16) + : +- ReusedExchange (19) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometFilter (23) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (22) + : +- ReusedExchange (24) + +- ReusedExchange (27) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#5, i_category#6] Condition : isnotnull(i_item_sk#5) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] +Arguments: [i_item_sk#5, i_category#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#6, 50, true, false, true) AS i_category#7] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#5, i_category#7] +Arguments: [i_item_sk#5, i_category#7] + +(7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [2]: [i_item_sk#5, i_category#6] +Right output [2]: [i_item_sk#5, i_category#7] Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight -(7) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(9) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(11) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Right output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [ss_sold_date_sk#4], [d_date_sk#7], Inner, BuildRight - -(12) CometProject -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12], [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Arguments: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] - -(14) CometFilter -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) - -(15) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#17, i_category#18] - -(16) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Right output [2]: [i_item_sk#17, i_category#18] -Arguments: [ws_item_sk#13], [i_item_sk#17], Inner, BuildRight - -(17) CometProject -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] -Arguments: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18], [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] - -(18) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] - -(19) CometBroadcastHashJoin -Left output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Right output [3]: [d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [ws_sold_date_sk#16], [d_date_sk#19], Inner, BuildRight - -(20) CometProject -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24], [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] - -(21) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Arguments: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] - -(22) CometFilter -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) - -(23) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#29, i_category#30] - -(24) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Right output [2]: [i_item_sk#29, i_category#30] -Arguments: [cs_item_sk#26], [i_item_sk#29], Inner, BuildRight - -(25) CometProject -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] -Arguments: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30], [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] - -(26) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] - -(27) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Right output [3]: [d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [cs_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight - -(28) CometProject -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36], [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] - -(29) CometUnion -Child 0 Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Child 1 Input [6]: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24] -Child 2 Input [6]: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36] - -(30) CometHashAggregate -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] - -(31) CometExchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] - -(33) CometTakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#10 ASC NULLS FIRST,col_name#11 ASC NULLS FIRST,d_year#8 ASC NULLS FIRST,d_qoy#9 ASC NULLS FIRST,i_category#6 ASC NULLS FIRST], output=[channel#10,col_name#11,d_year#8,d_qoy#9,i_category#6,sales_cnt#39,sales_amt#40]), [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40], 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] - -(34) ColumnarToRow [codegen id : 1] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#7] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : isnotnull(d_date_sk#8) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8, d_year#9, d_qoy#10] + +(12) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] +Right output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [ss_sold_date_sk#4], [d_date_sk#8], Inner, BuildRight + +(13) CometProject +Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7, d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13], [store AS channel#11, ss_store_sk#2 AS col_name#12, d_year#9, d_qoy#10, i_category#7, ss_ext_sales_price#3 AS ext_sales_price#13] + +(14) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(15) CometFilter +Input [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Condition : (isnull(ws_ship_customer_sk#15) AND isnotnull(ws_item_sk#14)) + +(16) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#18, i_category#19] + +(17) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Right output [2]: [i_item_sk#18, i_category#19] +Arguments: [ws_item_sk#14], [i_item_sk#18], Inner, BuildRight + +(18) CometProject +Input [6]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_item_sk#18, i_category#19] +Arguments: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19], [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] + +(19) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#20, d_year#21, d_qoy#22] + +(20) CometBroadcastHashJoin +Left output [4]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] +Right output [3]: [d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight + +(21) CometProject +Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] + +(22) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] +PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(23) CometFilter +Input [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Condition : (isnull(cs_ship_addr_sk#26) AND isnotnull(cs_item_sk#27)) + +(24) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#30, i_category#31] + +(25) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Right output [2]: [i_item_sk#30, i_category#31] +Arguments: [cs_item_sk#27], [i_item_sk#30], Inner, BuildRight + +(26) CometProject +Input [6]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29, i_item_sk#30, i_category#31] +Arguments: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31], [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] + +(27) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#32, d_year#33, d_qoy#34] + +(28) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] +Right output [3]: [d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [cs_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight + +(29) CometProject +Input [7]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31, d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37], [catalog AS channel#35, cs_ship_addr_sk#26 AS col_name#36, d_year#33, d_qoy#34, i_category#31, cs_ext_sales_price#28 AS ext_sales_price#37] + +(30) CometUnion +Child 0 Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Child 1 Input [6]: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25] +Child 2 Input [6]: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37] + +(31) CometHashAggregate +Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#13))] + +(32) CometExchange +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Arguments: hashpartitioning(channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(33) CometHashAggregate +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#13))] + +(34) CometTakeOrderedAndProject +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#11 ASC NULLS FIRST,col_name#12 ASC NULLS FIRST,d_year#9 ASC NULLS FIRST,d_qoy#10 ASC NULLS FIRST,i_category#7 ASC NULLS FIRST], output=[channel#11,col_name#12,d_year#9,d_qoy#10,i_category#7,sales_cnt#40,sales_amt#41]), [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41], 100, 0, [channel#11 ASC NULLS FIRST, col_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, i_category#7 ASC NULLS FIRST], [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] + +(35) CometColumnarToRow [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/extended.txt new file mode 100644 index 0000000000..cbb627c528 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/extended.txt @@ -0,0 +1,47 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 44 out of 44 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/simplified.txt index 0b641cea7d..abb482c3d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/simplified.txt @@ -1,29 +1,30 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count,sum,count(1),sum(UnscaledValue(ext_sales_price))] + CometHashAggregate [count,sum] [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count(1),sum(UnscaledValue(ext_sales_price))] CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 - CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum,ext_sales_price] + CometHashAggregate [ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,count,sum] CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category] #2 - CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] + CometProject [i_category] [i_item_sk,i_category] + CometFilter [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] @@ -31,6 +32,6 @@ WholeStageCodegen (1) CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/explain.txt index 2c6d00edda..476f7e4df0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/explain.txt @@ -1,70 +1,109 @@ == Physical Plan == -TakeOrderedAndProject (53) -+- * HashAggregate (52) - +- Exchange (51) - +- * HashAggregate (50) - +- * Expand (49) - +- Union (48) - :- * ColumnarToRow (22) - : +- CometProject (21) - : +- CometBroadcastHashJoin (20) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - :- * Project (41) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (40) - : :- BroadcastExchange (31) - : : +- * ColumnarToRow (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (23) - : : +- ReusedExchange (24) - : +- * ColumnarToRow (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (32) - : +- ReusedExchange (33) - +- * ColumnarToRow (47) - +- CometProject (46) - +- CometBroadcastHashJoin (45) - :- CometHashAggregate (43) - : +- ReusedExchange (42) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (85) ++- * HashAggregate (84) + +- * CometColumnarToRow (83) + +- CometColumnarExchange (82) + +- * HashAggregate (81) + +- * Expand (80) + +- Union (79) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (28) + : +- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (21) + : : +- CometBroadcastHashJoin (20) + : : :- CometFilter (18) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (17) + : : +- ReusedExchange (19) + : +- ReusedExchange (22) + :- * Project (50) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : :- BroadcastExchange (40) + : : +- * CometColumnarToRow (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (32) + : : +- ReusedExchange (33) + : +- * CometColumnarToRow (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometNativeScan parquet spark_catalog.default.catalog_returns (41) + : +- ReusedExchange (42) + +- * CometColumnarToRow (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometFilter (52) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (51) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometNativeScan parquet spark_catalog.default.web_page (56) + +- CometBroadcastExchange (75) + +- CometHashAggregate (74) + +- CometExchange (73) + +- CometHashAggregate (72) + +- CometProject (71) + +- CometBroadcastHashJoin (70) + :- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometFilter (65) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (64) + : +- ReusedExchange (66) + +- ReusedExchange (69) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_date#6] @@ -87,9 +126,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [1]: [s_store_sk#7] @@ -122,156 +164,312 @@ Input [3]: [s_store_sk#7, sum#8, sum#9] Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [s_store_sk#10, sum#11, sum#12] +(17) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct -(18) CometHashAggregate -Input [3]: [s_store_sk#10, sum#11, sum#12] -Keys [1]: [s_store_sk#10] -Functions [2]: [sum(UnscaledValue(sr_return_amt#13)), sum(UnscaledValue(sr_net_loss#14))] +(18) CometFilter +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) -(19) CometBroadcastExchange -Input [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#10, returns#15, profit_loss#16] +(19) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#14] (20) CometBroadcastHashJoin -Left output [3]: [s_store_sk#7, sales#17, profit#18] -Right output [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#7], [s_store_sk#10], LeftOuter, BuildRight +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight (21) CometProject -Input [6]: [s_store_sk#7, sales#17, profit#18, s_store_sk#10, returns#15, profit_loss#16] -Arguments: [sales#17, returns#19, profit#20, channel#21, id#22], [sales#17, coalesce(returns#15, 0.00) AS returns#19, (profit#18 - coalesce(profit_loss#16, 0.00)) AS profit#20, store channel AS channel#21, s_store_sk#7 AS id#22] +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] -(22) ColumnarToRow [codegen id : 1] -Input [5]: [sales#17, returns#19, profit#20, channel#21, id#22] +(22) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#15] -(23) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] +(23) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight -(24) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#27] +(24) CometProject +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] -(25) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [cs_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight +(25) CometHashAggregate +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] -(26) CometProject -Input [5]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26, d_date_sk#27] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25], [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] +(26) CometExchange +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate -Input [3]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#24)), partial_sum(UnscaledValue(cs_net_profit#25))] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] -(28) CometExchange -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Arguments: hashpartitioning(cs_call_center_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometBroadcastExchange +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] -(29) CometHashAggregate -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#24)), sum(UnscaledValue(cs_net_profit#25))] +(29) CometBroadcastHashJoin +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight -(30) ColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] +(30) CometProject +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [sales#20, returns#22, profit#23, channel#24, id#25], [sales#20, coalesce(returns#18, 0.00) AS returns#22, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#23, store channel AS channel#24, s_store_sk#7 AS id#25] -(31) BroadcastExchange -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] -Arguments: IdentityBroadcastMode, [plan_id=3] +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] -(32) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Arguments: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] +(32) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] +ReadSchema: struct (33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#35] +Output [1]: [d_date_sk#30] (34) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Right output [1]: [d_date_sk#35] -Arguments: [cr_returned_date_sk#34], [d_date_sk#35], Inner, BuildRight +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (35) CometProject -Input [4]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34, d_date_sk#35] -Arguments: [cr_return_amount#32, cr_net_loss#33], [cr_return_amount#32, cr_net_loss#33] +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] (36) CometHashAggregate -Input [2]: [cr_return_amount#32, cr_net_loss#33] -Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#32)), partial_sum(UnscaledValue(cr_net_loss#33))] +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] (37) CometExchange -Input [2]: [sum#36, sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate -Input [2]: [sum#36, sum#37] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] + +(39) CometColumnarToRow [codegen id : 2] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] + +(40) BroadcastExchange +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] +Arguments: IdentityBroadcastMode, [plan_id=4] + +(41) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] +ReadSchema: struct + +(42) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#38] + +(43) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight + +(44) CometProject +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] + +(45) CometHashAggregate +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#32)), sum(UnscaledValue(cr_net_loss#33))] - -(39) ColumnarToRow -Input [2]: [returns#38, profit_loss#39] - -(40) BroadcastNestedLoopJoin [codegen id : 3] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 3] -Output [5]: [sales#30, returns#38, (profit#31 - profit_loss#39) AS profit#40, catalog channel AS channel#41, cs_call_center_sk#23 AS id#42] -Input [5]: [cs_call_center_sk#23, sales#30, profit#31, returns#38, profit_loss#39] - -(42) ReusedExchange [Reuses operator id: 15] -Output [3]: [wp_web_page_sk#43, sum#44, sum#45] - -(43) CometHashAggregate -Input [3]: [wp_web_page_sk#43, sum#44, sum#45] -Keys [1]: [wp_web_page_sk#43] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#46)), sum(UnscaledValue(ws_net_profit#47))] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] -(44) ReusedExchange [Reuses operator id: 19] -Output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] +(46) CometExchange +Input [2]: [sum#39, sum#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(45) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#43, sales#51, profit#52] -Right output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [wp_web_page_sk#43], [wp_web_page_sk#48], LeftOuter, BuildRight - -(46) CometProject -Input [6]: [wp_web_page_sk#43, sales#51, profit#52, wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [sales#51, returns#53, profit#54, channel#55, id#56], [sales#51, coalesce(returns#49, 0.00) AS returns#53, (profit#52 - coalesce(profit_loss#50, 0.00)) AS profit#54, web channel AS channel#55, wp_web_page_sk#43 AS id#56] - -(47) ColumnarToRow [codegen id : 4] -Input [5]: [sales#51, returns#53, profit#54, channel#55, id#56] - -(48) Union - -(49) Expand [codegen id : 5] -Input [5]: [sales#17, returns#19, profit#20, channel#21, id#22] -Arguments: [[sales#17, returns#19, profit#20, channel#21, id#22, 0], [sales#17, returns#19, profit#20, channel#21, null, 1], [sales#17, returns#19, profit#20, null, null, 3]], [sales#17, returns#19, profit#20, channel#57, id#58, spark_grouping_id#59] - -(50) HashAggregate [codegen id : 5] -Input [6]: [sales#17, returns#19, profit#20, channel#57, id#58, spark_grouping_id#59] -Keys [3]: [channel#57, id#58, spark_grouping_id#59] -Functions [3]: [partial_sum(sales#17), partial_sum(returns#19), partial_sum(profit#20)] -Aggregate Attributes [6]: [sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Results [9]: [channel#57, id#58, spark_grouping_id#59, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] +(47) CometHashAggregate +Input [2]: [sum#39, sum#40] +Keys: [] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] -(51) Exchange -Input [9]: [channel#57, id#58, spark_grouping_id#59, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(channel#57, id#58, spark_grouping_id#59, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(48) CometColumnarToRow +Input [2]: [returns#41, profit_loss#42] -(52) HashAggregate [codegen id : 6] -Input [9]: [channel#57, id#58, spark_grouping_id#59, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [3]: [channel#57, id#58, spark_grouping_id#59] -Functions [3]: [sum(sales#17), sum(returns#19), sum(profit#20)] -Aggregate Attributes [3]: [sum(sales#17)#72, sum(returns#19)#73, sum(profit#20)#74] -Results [5]: [channel#57, id#58, sum(sales#17)#72 AS sales#75, sum(returns#19)#73 AS returns#76, sum(profit#20)#74 AS profit#77] +(49) BroadcastNestedLoopJoin [codegen id : 3] +Join type: Inner +Join condition: None -(53) TakeOrderedAndProject -Input [5]: [channel#57, id#58, sales#75, returns#76, profit#77] -Arguments: 100, [channel#57 ASC NULLS FIRST, id#58 ASC NULLS FIRST], [channel#57, id#58, sales#75, returns#76, profit#77] +(50) Project [codegen id : 3] +Output [5]: [sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#43, catalog channel AS channel#44, cs_call_center_sk#26 AS id#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] + +(51) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] +PushedFilters: [IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(52) CometFilter +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) + +(53) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#50] + +(54) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight + +(55) CometProject +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] + +(56) CometNativeScan parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(57) CometFilter +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) + +(58) CometBroadcastExchange +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] + +(59) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight + +(60) CometProject +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] + +(61) CometHashAggregate +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] + +(62) CometExchange +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(63) CometHashAggregate +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] + +(64) CometNativeScan parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct + +(65) CometFilter +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) + +(66) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(67) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(68) CometProject +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] + +(69) ReusedExchange [Reuses operator id: 58] +Output [1]: [wp_web_page_sk#59] + +(70) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(71) CometProject +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] + +(72) CometHashAggregate +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] + +(73) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(74) CometHashAggregate +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] + +(75) CometBroadcastExchange +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] + +(76) CometBroadcastHashJoin +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight + +(77) CometProject +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [sales#64, returns#66, profit#67, channel#68, id#69], [sales#64, coalesce(returns#62, 0.00) AS returns#66, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#67, web channel AS channel#68, wp_web_page_sk#51 AS id#69] + +(78) CometColumnarToRow [codegen id : 4] +Input [5]: [sales#64, returns#66, profit#67, channel#68, id#69] + +(79) Union + +(80) Expand [codegen id : 5] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] +Arguments: [[sales#20, returns#22, profit#23, channel#24, id#25, 0], [sales#20, returns#22, profit#23, channel#24, null, 1], [sales#20, returns#22, profit#23, null, null, 3]], [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] + +(81) HashAggregate [codegen id : 5] +Input [6]: [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#22), partial_sum(profit#23)] +Aggregate Attributes [6]: [sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78] +Results [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] + +(82) CometColumnarExchange +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Arguments: hashpartitioning(channel#70, id#71, spark_grouping_id#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(83) CometColumnarToRow [codegen id : 6] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] + +(84) HashAggregate [codegen id : 6] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [sum(sales#20), sum(returns#22), sum(profit#23)] +Aggregate Attributes [3]: [sum(sales#20)#85, sum(returns#22)#86, sum(profit#23)#87] +Results [5]: [channel#70, id#71, sum(sales#20)#85 AS sales#88, sum(returns#22)#86 AS returns#89, sum(profit#23)#87 AS profit#90] + +(85) TakeOrderedAndProject +Input [5]: [channel#70, id#71, sales#88, returns#89, profit#90] +Arguments: 100, [channel#70 ASC NULLS FIRST, id#71 ASC NULLS FIRST], [channel#70, id#71, sales#88, returns#89, profit#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/extended.txt new file mode 100644 index 0000000000..f857003b9a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/extended.txt @@ -0,0 +1,106 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Union + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- Project + : +- BroadcastNestedLoopJoin + : :- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometColumnarToRow + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 91 out of 99 eligible operators (91%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/simplified.txt index c41db2cfab..165dc44350 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/simplified.txt @@ -1,66 +1,98 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (6) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometExchange [s_store_sk] #2 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #4 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #5 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - ReusedExchange [s_store_sk,sum,sum] #2 - WholeStageCodegen (3) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (5) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #7 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ColumnarToRow + CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [s_store_sk,sales,profit,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #2 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #4 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #5 + CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #6 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #4 + WholeStageCodegen (3) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #8 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - ReusedExchange [wp_web_page_sk,sum,sum] #2 - ReusedExchange [wp_web_page_sk,returns,profit_loss] #5 + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #8 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #9 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #10 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #12 + CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #13 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/explain.txt index ae6446061d..fdcf5824de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * ColumnarToRow (46) - +- CometSortMergeJoin (45) - :- CometProject (27) - : +- CometSortMergeJoin (26) +TakeOrderedAndProject (67) ++- * Project (66) + +- * CometColumnarToRow (65) + +- CometSortMergeJoin (64) + :- CometProject (43) + : +- CometSortMergeJoin (42) : :- CometSort (21) : : +- CometHashAggregate (20) : : +- CometExchange (19) @@ -17,41 +17,64 @@ TakeOrderedAndProject (48) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) : : +- CometBroadcastExchange (15) : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometSort (25) - : +- CometFilter (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - +- CometSort (44) - +- CometFilter (43) - +- CometHashAggregate (42) - +- CometExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometFilter (35) - : +- CometSortMergeJoin (34) - : :- CometSort (31) - : : +- CometExchange (30) - : : +- CometFilter (29) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : +- CometSort (33) - : +- ReusedExchange (32) - +- ReusedExchange (37) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : +- CometSort (41) + : +- CometFilter (40) + : +- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (33) + : : +- CometFilter (32) + : : +- CometSortMergeJoin (31) + : : :- CometSort (25) + : : : +- CometExchange (24) + : : : +- CometFilter (23) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (22) + : : +- CometSort (30) + : : +- CometExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (26) + : +- ReusedExchange (34) + +- CometSort (63) + +- CometFilter (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometFilter (54) + : +- CometSortMergeJoin (53) + : :- CometSort (47) + : : +- CometExchange (46) + : : +- CometFilter (45) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (44) + : +- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometFilter (49) + : +- CometNativeScan parquet spark_catalog.default.catalog_returns (48) + +- ReusedExchange (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -65,9 +88,12 @@ Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (6) CometFilter Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] @@ -98,9 +124,12 @@ Condition : isnull(sr_ticket_number#9) Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [2]: [d_date_sk#11, d_year#12] @@ -137,114 +166,208 @@ Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(22) ReusedExchange [Reuses operator id: 19] -Output [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] +(22) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct -(23) CometHashAggregate -Input [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] -Keys [3]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +(23) CometFilter +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) -(24) CometFilter -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Condition : (coalesce(ws_qty#31, 0) > 0) +(24) CometExchange +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometSort -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33], [ws_sold_year#29 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#30 ASC NULLS FIRST] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(26) CometSortMergeJoin -Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Right output [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30], Inner +(26) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct -(27) CometProject -Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] +(27) CometFilter +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] +(28) CometProject +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] -(29) CometFilter -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Condition : (isnotnull(cs_item_sk#35) AND isnotnull(cs_bill_customer_sk#34)) +(29) CometExchange +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(30) CometExchange -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_order_number#36, cs_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(30) CometSort +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] -(31) CometSort -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_order_number#36 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST] +(31) CometSortMergeJoin +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter -(32) ReusedExchange [Reuses operator id: 8] -Output [2]: [cr_item_sk#41, cr_order_number#42] +(32) CometFilter +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) -(33) CometSort -Input [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cr_item_sk#41, cr_order_number#42], [cr_order_number#42 ASC NULLS FIRST, cr_item_sk#41 ASC NULLS FIRST] +(33) CometProject +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] -(34) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cs_order_number#36, cs_item_sk#35], [cr_order_number#42, cr_item_sk#41], LeftOuter +(34) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#30, d_year#31] -(35) CometFilter -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Condition : isnull(cr_order_number#42) +(35) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight (36) CometProject -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] - -(37) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#43, d_year#44] - -(38) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [cs_sold_date_sk#40], [d_date_sk#43], Inner, BuildRight - -(39) CometProject -Input [8]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, d_date_sk#43, d_year#44] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] - -(40) CometHashAggregate -Input [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [partial_sum(cs_quantity#37), partial_sum(UnscaledValue(cs_wholesale_cost#38)), partial_sum(UnscaledValue(cs_sales_price#39))] - -(41) CometExchange -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Arguments: hashpartitioning(d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(42) CometHashAggregate -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [sum(cs_quantity#37), sum(UnscaledValue(cs_wholesale_cost#38)), sum(UnscaledValue(cs_sales_price#39))] - -(43) CometFilter -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Condition : (coalesce(cs_qty#50, 0) > 0) - -(44) CometSort -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52], [cs_sold_year#48 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST, cs_customer_sk#49 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] -Right output [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49], Inner - -(46) ColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(47) Project [codegen id : 1] -Output [12]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#31 + cs_qty#50), 1) as double)))), 2) AS ratio#53, ss_qty#17 AS store_qty#54, ss_wc#18 AS store_wholesale_cost#55, ss_sp#19 AS store_sales_price#56, (coalesce(ws_qty#31, 0) + coalesce(cs_qty#50, 0)) AS other_chan_qty#57, (coalesce(ws_wc#32, 0.00) + coalesce(cs_wc#51, 0.00)) AS other_chan_wholesale_cost#58, (coalesce(ws_sp#33, 0.00) + coalesce(cs_sp#52, 0.00)) AS other_chan_sales_price#59, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, cs_qty#50] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(48) TakeOrderedAndProject -Input [12]: [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, cs_qty#50] -Arguments: 100, [ratio#53 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#57 ASC NULLS FIRST, other_chan_wholesale_cost#58 ASC NULLS FIRST, other_chan_sales_price#59 ASC NULLS FIRST, round((cast(ss_qty#17 as double) / cast(coalesce((ws_qty#31 + cs_qty#50), 1) as double)), 2) ASC NULLS FIRST], [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59] +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] + +(37) CometHashAggregate +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] + +(38) CometExchange +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometHashAggregate +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] + +(40) CometFilter +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) + +(41) CometSort +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] + +(42) CometSortMergeJoin +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner + +(43) CometProject +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] + +(44) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(45) CometFilter +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) + +(46) CometExchange +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(47) CometSort +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] + +(48) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(49) CometFilter +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) + +(50) CometProject +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] + +(51) CometExchange +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(52) CometSort +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] + +(53) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter + +(54) CometFilter +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) + +(55) CometProject +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] + +(56) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#50, d_year#51] + +(57) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight + +(58) CometProject +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] + +(59) CometHashAggregate +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] + +(60) CometExchange +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(61) CometHashAggregate +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] + +(62) CometFilter +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) + +(63) CometSort +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] + +(64) CometSortMergeJoin +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner + +(65) CometColumnarToRow [codegen id : 1] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(66) Project [codegen id : 1] +Output [12]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(67) TakeOrderedAndProject +Input [12]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Arguments: 100, [ratio#60 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, round((cast(ss_qty#17 as double) / cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)), 2) ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/extended.txt new file mode 100644 index 0000000000..f180fb1878 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/extended.txt @@ -0,0 +1,73 @@ + TakeOrderedAndProject [COMET: ] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding] + +- CometColumnarToRow + +- CometSortMergeJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 68 out of 70 eligible operators (97%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/simplified.txt index 060ad787bd..4793204ab0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/simplified.txt @@ -1,15 +1,15 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] - ColumnarToRow + CometColumnarToRow InputAdapter CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] - CometHashAggregate [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum,sum,sum,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 - CometHashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum,ss_quantity,ss_wholesale_cost,ss_sales_price] + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] @@ -18,33 +18,52 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometHashAggregate [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum,sum,sum,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - ReusedExchange [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] #1 + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometHashAggregate [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum,sum,sum,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #5 - CometHashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum,cs_quantity,cs_wholesale_cost,cs_sales_price] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #6 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number] #3 + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/explain.txt index f22d1d9714..eacdb29f86 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometProject (28) - +- CometBroadcastHashJoin (27) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometProject (29) + +- CometBroadcastHashJoin (28) :- CometHashAggregate (23) : +- CometExchange (22) : +- CometHashAggregate (21) @@ -13,35 +13,43 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (26) - +- CometFilter (25) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (27) + +- CometProject (26) + +- CometFilter (25) + +- CometNativeScan parquet spark_catalog.default.customer (24) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] @@ -64,9 +72,12 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Arguments: [s_store_sk#12, s_number_employees#13, s_city#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] @@ -89,9 +100,12 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] @@ -128,31 +142,38 @@ Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#1 Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -(24) CometNativeScan: `spark_catalog`.`default`.`customer` +(24) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (25) CometFilter Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] Condition : isnotnull(c_customer_sk#20) -(26) CometBroadcastExchange +(26) CometProject Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24], [c_customer_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#21, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#22, 30, true, false, true) AS c_last_name#24] -(27) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#23, profit#24] -Right output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(27) CometBroadcastExchange +Input [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24] + +(28) CometBroadcastHashJoin +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26] +Right output [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] Arguments: [ss_customer_sk#1], [c_customer_sk#20], Inner, BuildRight -(28) CometProject -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#23, profit#24, c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14], [c_last_name#22, c_first_name#21, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14] +(29) CometProject +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26, c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14], [c_last_name#24, c_first_name#23, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] -(29) CometTakeOrderedAndProject -Input [7]: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#22 ASC NULLS FIRST,c_first_name#21 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#24 ASC NULLS FIRST], output=[c_last_name#22,c_first_name#21,substr(s_city, 1, 30)#25,ss_ticket_number#5,amt#23,profit#24]), [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24], 100, [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#24 ASC NULLS FIRST], [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24] +(30) CometTakeOrderedAndProject +Input [7]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#26 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,substr(s_city, 1, 30)#27,ss_ticket_number#5,amt#25,profit#26]), [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26], 100, 0, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#26 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] -(30) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24] +(31) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/extended.txt new file mode 100644 index 0000000000..178dbb8e72 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/simplified.txt index 8c5b3a2e53..7612d7f5ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] CometProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,s_city,amt,profit,c_customer_sk,c_first_name,c_last_name] - CometHashAggregate [ss_ticket_number,ss_customer_sk,s_city,amt,profit,ss_addr_sk,sum,sum,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] + CometHashAggregate [ss_addr_sk,sum,sum] [ss_ticket_number,ss_customer_sk,s_city,amt,profit,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum,ss_coupon_amt,ss_net_profit] + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] @@ -14,19 +14,20 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] CometBroadcastExchange [s_store_sk,s_city] #3 CometProject [s_store_sk,s_city] CometFilter [s_store_sk,s_number_employees,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_number_employees,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/explain.txt index c65d269ef2..619df5d251 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/explain.txt @@ -1,59 +1,68 @@ == Physical Plan == -* ColumnarToRow (42) -+- CometTakeOrderedAndProject (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (35) - +- CometHashAggregate (34) - +- CometExchange (33) - +- CometHashAggregate (32) - +- CometBroadcastHashJoin (31) - :- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (14) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometFilter (28) - +- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (17) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (37) + +- CometHashAggregate (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometBroadcastHashJoin (33) + :- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.customer_address (15) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.customer_address (18) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.customer (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] @@ -76,148 +85,167 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) +Condition : (isnotnull(s_store_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true))) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true) AS s_zip#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [ss_net_profit#2, s_store_name#8, s_zip#9], [ss_net_profit#2, s_store_name#8, s_zip#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [1]: [ca_zip#10] -Arguments: [ca_zip#10] - -(15) CometFilter -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) - -(16) CometProject -Input [1]: [ca_zip#10] -Arguments: [ca_zip#11], [substr(ca_zip#10, 1, 5) AS ca_zip#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#12, ca_zip#13] -Arguments: [ca_address_sk#12, ca_zip#13] - -(18) CometFilter -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) +(14) CometProject +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] -(19) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14, c_preferred_cust_flag#15] +(15) CometNativeScan parquet spark_catalog.default.customer_address +Output [1]: [ca_zip#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +ReadSchema: struct -(20) CometFilter -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) - -(21) CometProject -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14], [c_current_addr_sk#14] - -(22) CometBroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: [c_current_addr_sk#14] - -(23) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#12, ca_zip#13] -Right output [1]: [c_current_addr_sk#14] -Arguments: [ca_address_sk#12], [c_current_addr_sk#14], Inner, BuildRight +(16) CometFilter +Input [1]: [ca_zip#11] +Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5))) -(24) CometProject -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] -Arguments: [ca_zip#13], [ca_zip#13] +(17) CometProject +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) AS ca_zip#12] + +(18) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) + +(20) CometProject +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#14, 10, true, false, true) AS ca_zip#15] + +(21) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#17, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#16)) + +(23) CometProject +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] + +(24) CometBroadcastExchange +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] + +(25) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight + +(26) CometProject +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] -(25) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +(27) CometHashAggregate +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] -(26) CometExchange -Input [2]: [ca_zip#13, count#16] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(28) CometExchange +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [2]: [ca_zip#13, count#16] -Keys [1]: [ca_zip#13] +(29) CometHashAggregate +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] -(28) CometFilter -Input [2]: [ca_zip#17, cnt#18] -Condition : (cnt#18 > 10) +(30) CometFilter +Input [2]: [ca_zip#19, cnt#20] +Condition : (cnt#20 > 10) -(29) CometProject -Input [2]: [ca_zip#17, cnt#18] -Arguments: [ca_zip#17], [ca_zip#17] +(31) CometProject +Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] -(30) CometBroadcastExchange -Input [1]: [ca_zip#17] -Arguments: [ca_zip#17] +(32) CometBroadcastExchange +Input [1]: [ca_zip#19] +Arguments: [ca_zip#19] -(31) CometBroadcastHashJoin -Left output [1]: [ca_zip#11] -Right output [1]: [ca_zip#17] -Arguments: [coalesce(ca_zip#11, ), isnull(ca_zip#11)], [coalesce(ca_zip#17, ), isnull(ca_zip#17)], LeftSemi, BuildRight +(33) CometBroadcastHashJoin +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight -(32) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(34) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -(33) CometExchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(35) CometExchange +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(36) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -(35) CometBroadcastExchange -Input [1]: [ca_zip#11] -Arguments: [ca_zip#11] +(37) CometBroadcastExchange +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] -(36) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Right output [1]: [ca_zip#11] -Arguments: [substr(s_zip#9, 1, 2)], [substr(ca_zip#11, 1, 2)], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight -(37) CometProject -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] +(39) CometProject +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] -(38) CometHashAggregate +(40) CometHashAggregate Input [2]: [ss_net_profit#2, s_store_name#8] Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -(39) CometExchange -Input [2]: [s_store_name#8, sum#19] +(41) CometExchange +Input [2]: [s_store_name#8, sum#21] Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(40) CometHashAggregate -Input [2]: [s_store_name#8, sum#19] +(42) CometHashAggregate +Input [2]: [s_store_name#8, sum#21] Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -(41) CometTakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#20] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#20]), [s_store_name#8, sum(ss_net_profit)#20], 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#20] +(43) CometTakeOrderedAndProject +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#22]), [s_store_name#8, sum(ss_net_profit)#22], 100, 0, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] -(42) ColumnarToRow [codegen id : 1] -Input [2]: [s_store_name#8, sum(ss_net_profit)#20] +(44) CometColumnarToRow [codegen id : 1] +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/extended.txt new file mode 100644 index 0000000000..c84e490026 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/extended.txt @@ -0,0 +1,46 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 43 out of 43 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/simplified.txt index 6e915b2cee..123d33addd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - CometHashAggregate [s_store_name,sum(ss_net_profit),sum,sum(UnscaledValue(ss_net_profit))] + CometHashAggregate [sum] [s_store_name,sum(ss_net_profit),sum(UnscaledValue(ss_net_profit))] CometExchange [s_store_name] #1 - CometHashAggregate [s_store_name,sum,ss_net_profit] + CometHashAggregate [ss_net_profit] [s_store_name,sum] CometProject [ss_net_profit,s_store_name] CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] CometProject [ss_net_profit,s_store_name,s_zip] @@ -12,14 +12,15 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [ca_zip] #4 CometHashAggregate [ca_zip] CometExchange [ca_zip] #5 @@ -27,18 +28,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_zip] CometBroadcastExchange [ca_zip] #6 CometProject [ca_zip] CometFilter [ca_zip,cnt] - CometHashAggregate [ca_zip,cnt,ca_zip,count,count(1)] + CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] CometExchange [ca_zip] #7 CometHashAggregate [ca_zip,count] CometProject [ca_zip] CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [c_current_addr_sk] #8 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_current_addr_sk,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/explain.txt index 91f9e5a088..dda2ae6aa9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/explain.txt @@ -1,80 +1,116 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Expand (67) - +- Union (66) - :- * HashAggregate (38) - : +- * ColumnarToRow (37) - : +- CometExchange (36) - : +- CometHashAggregate (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometProject (17) - : : : : +- CometBroadcastHashJoin (16) - : : : : :- CometProject (11) - : : : : : +- CometSortMergeJoin (10) - : : : : : :- CometSort (4) - : : : : : : +- CometExchange (3) - : : : : : : +- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometSort (9) - : : : : : +- CometExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : : : +- CometBroadcastExchange (15) - : : : : +- CometProject (14) - : : : : +- CometFilter (13) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - : : : +- CometBroadcastExchange (20) - : : : +- CometFilter (19) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (18) - : : +- CometBroadcastExchange (26) - : : +- CometProject (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (23) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (29) - :- * HashAggregate (62) - : +- * ColumnarToRow (61) - : +- CometExchange (60) - : +- CometHashAggregate (59) - : +- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometProject (49) - : : : : +- CometBroadcastHashJoin (48) - : : : : :- CometProject (46) - : : : : : +- CometSortMergeJoin (45) - : : : : : :- CometSort (42) - : : : : : : +- CometExchange (41) - : : : : : : +- CometFilter (40) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : +- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- ReusedExchange (47) - : : : +- ReusedExchange (50) - : : +- ReusedExchange (53) - : +- ReusedExchange (56) - +- * HashAggregate (65) - +- * ColumnarToRow (64) - +- ReusedExchange (63) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (103) ++- CometTakeOrderedAndProject (102) + +- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometExpand (98) + +- CometUnion (97) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (17) + : : : : +- CometBroadcastHashJoin (16) + : : : : :- CometProject (11) + : : : : : +- CometSortMergeJoin (10) + : : : : : :- CometSort (4) + : : : : : : +- CometExchange (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometSort (9) + : : : : : +- CometExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : : : +- CometBroadcastExchange (15) + : : : : +- CometProject (14) + : : : : +- CometFilter (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : : : +- CometBroadcastExchange (21) + : : : +- CometProject (20) + : : : +- CometFilter (19) + : : : +- CometNativeScan parquet spark_catalog.default.store (18) + : : +- CometBroadcastExchange (27) + : : +- CometProject (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.item (24) + : +- CometBroadcastExchange (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometNativeScan parquet spark_catalog.default.promotion (30) + :- CometHashAggregate (67) + : +- CometExchange (66) + : +- CometHashAggregate (65) + : +- CometProject (64) + : +- CometBroadcastHashJoin (63) + : :- CometProject (61) + : : +- CometBroadcastHashJoin (60) + : : :- CometProject (58) + : : : +- CometBroadcastHashJoin (57) + : : : :- CometProject (52) + : : : : +- CometBroadcastHashJoin (51) + : : : : :- CometProject (49) + : : : : : +- CometSortMergeJoin (48) + : : : : : :- CometSort (42) + : : : : : : +- CometExchange (41) + : : : : : : +- CometFilter (40) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (39) + : : : : : +- CometSort (47) + : : : : : +- CometExchange (46) + : : : : : +- CometProject (45) + : : : : : +- CometFilter (44) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (43) + : : : : +- ReusedExchange (50) + : : : +- CometBroadcastExchange (56) + : : : +- CometProject (55) + : : : +- CometFilter (54) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (53) + : : +- ReusedExchange (59) + : +- ReusedExchange (62) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometBroadcastHashJoin (86) + : : :- CometProject (81) + : : : +- CometBroadcastHashJoin (80) + : : : :- CometProject (78) + : : : : +- CometSortMergeJoin (77) + : : : : :- CometSort (71) + : : : : : +- CometExchange (70) + : : : : : +- CometFilter (69) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (68) + : : : : +- CometSort (76) + : : : : +- CometExchange (75) + : : : : +- CometProject (74) + : : : : +- CometFilter (73) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (72) + : : : +- ReusedExchange (79) + : : +- CometBroadcastExchange (85) + : : +- CometProject (84) + : : +- CometFilter (83) + : : +- CometNativeScan parquet spark_catalog.default.web_site (82) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -88,9 +124,12 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (6) CometFilter Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] @@ -117,9 +156,12 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(12) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] +ReadSchema: struct (13) CometFilter Input [2]: [d_date_sk#13, d_date#14] @@ -142,233 +184,391 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(18) CometNativeScan: `spark_catalog`.`default`.`store` +(18) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (19) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(20) CometBroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] -(21) CometBroadcastHashJoin +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] +Right output [2]: [s_store_sk#15, s_store_id#17] Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(23) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17, i_current_price#18] - -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] - -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] - -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(29) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19, p_channel_tv#20] - -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] - -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] - -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight - -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(35) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) + +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] + +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] + +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(30) CometNativeScan parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) + +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] + +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] + +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight + +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -(36) CometExchange -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(37) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(38) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Keys [1]: [s_store_id#16] +(38) CometHashAggregate +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#26, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#26,17,2) AS sales#29, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27 AS returns#30, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28 AS profit#31, store channel AS channel#32, concat(store, s_store_id#16) AS id#33] -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] +(39) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct (40) CometFilter -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Condition : ((isnotnull(cs_catalog_page_sk#34) AND isnotnull(cs_item_sk#35)) AND isnotnull(cs_promo_sk#36)) +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : ((isnotnull(cs_catalog_page_sk#27) AND isnotnull(cs_item_sk#28)) AND isnotnull(cs_promo_sk#29)) (41) CometExchange -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_item_sk#35, cs_order_number#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: hashpartitioning(cs_item_sk#28, cs_order_number#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (42) CometSort -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40], [cs_item_sk#35 ASC NULLS FIRST, cs_order_number#37 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33], [cs_item_sk#28 ASC NULLS FIRST, cs_order_number#30 ASC NULLS FIRST] -(43) ReusedExchange [Reuses operator id: 8] -Output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] +(43) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct -(44) CometSort -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST] +(44) CometFilter +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Condition : (isnotnull(cr_item_sk#34) AND isnotnull(cr_order_number#35)) -(45) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Right output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_item_sk#35, cs_order_number#37], [cr_item_sk#41, cr_order_number#42], LeftOuter +(45) CometProject +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] -(46) CometProject -Input [11]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] +(46) CometExchange +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: hashpartitioning(cr_item_sk#34, cr_order_number#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(47) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#45] +(47) CometSort +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34 ASC NULLS FIRST, cr_order_number#35 ASC NULLS FIRST] -(48) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] -Right output [1]: [d_date_sk#45] -Arguments: [cs_sold_date_sk#40], [d_date_sk#45], Inner, BuildRight +(48) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_item_sk#28, cs_order_number#30], [cr_item_sk#34, cr_order_number#35], LeftOuter (49) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44, d_date_sk#45] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] +Input [11]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] -(50) ReusedExchange [Reuses operator id: 20] -Output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] +(50) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#39] (51) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] -Right output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_catalog_page_sk#34], [cp_catalog_page_sk#46], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] +Right output [1]: [d_date_sk#39] +Arguments: [cs_sold_date_sk#33], [d_date_sk#39], Inner, BuildRight (52) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37, d_date_sk#39] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] -(53) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#48] +(53) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct -(54) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [i_item_sk#48] -Arguments: [cs_item_sk#35], [i_item_sk#48], Inner, BuildRight +(54) CometFilter +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Condition : isnotnull(cp_catalog_page_sk#40) (55) CometProject -Input [8]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, i_item_sk#48] -Arguments: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42], [cp_catalog_page_sk#40, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#41, 16, true, false, true) AS cp_catalog_page_id#42] -(56) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#49] +(56) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42] (57) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [p_promo_sk#49] -Arguments: [cs_promo_sk#36], [p_promo_sk#49], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] +Right output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_catalog_page_sk#27], [cp_catalog_page_sk#40], Inner, BuildRight (58) CometProject -Input [7]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, p_promo_sk#49] -Arguments: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(59) CometHashAggregate -Input [5]: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#38)), partial_sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] - -(60) CometExchange -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Arguments: hashpartitioning(cp_catalog_page_id#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(61) ColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] - -(62) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#38)), sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#38))#55, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#38))#55,17,2) AS sales#58, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56 AS returns#59, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57 AS profit#60, catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#47) AS id#62] - -(63) ReusedExchange [Reuses operator id: 36] -Output [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(64) ColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(65) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [1]: [web_site_id#63] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#69)), sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#69))#73, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#69))#73,17,2) AS sales#76, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74 AS returns#77, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75 AS profit#78, web channel AS channel#79, concat(web_site, web_site_id#63) AS id#80] - -(66) Union - -(67) Expand [codegen id : 4] -Input [5]: [sales#29, returns#30, profit#31, channel#32, id#33] -Arguments: [[sales#29, returns#30, profit#31, channel#32, id#33, 0], [sales#29, returns#30, profit#31, channel#32, null, 1], [sales#29, returns#30, profit#31, null, null, 3]], [sales#29, returns#30, profit#31, channel#81, id#82, spark_grouping_id#83] - -(68) HashAggregate [codegen id : 4] -Input [6]: [sales#29, returns#30, profit#31, channel#81, id#82, spark_grouping_id#83] -Keys [3]: [channel#81, id#82, spark_grouping_id#83] -Functions [3]: [partial_sum(sales#29), partial_sum(returns#30), partial_sum(profit#31)] -Aggregate Attributes [6]: [sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] -Results [9]: [channel#81, id#82, spark_grouping_id#83, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95] - -(69) Exchange -Input [9]: [channel#81, id#82, spark_grouping_id#83, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Arguments: hashpartitioning(channel#81, id#82, spark_grouping_id#83, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(70) HashAggregate [codegen id : 5] -Input [9]: [channel#81, id#82, spark_grouping_id#83, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Keys [3]: [channel#81, id#82, spark_grouping_id#83] -Functions [3]: [sum(sales#29), sum(returns#30), sum(profit#31)] -Aggregate Attributes [3]: [sum(sales#29)#96, sum(returns#30)#97, sum(profit#31)#98] -Results [5]: [channel#81, id#82, sum(sales#29)#96 AS sales#99, sum(returns#30)#97 AS returns#100, sum(profit#31)#98 AS profit#101] - -(71) TakeOrderedAndProject -Input [5]: [channel#81, id#82, sales#99, returns#100, profit#101] -Arguments: 100, [channel#81 ASC NULLS FIRST, id#82 ASC NULLS FIRST], [channel#81, id#82, sales#99, returns#100, profit#101] +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(59) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#43] + +(60) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [i_item_sk#43] +Arguments: [cs_item_sk#28], [i_item_sk#43], Inner, BuildRight + +(61) CometProject +Input [8]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, i_item_sk#43] +Arguments: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(62) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#44] + +(63) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [p_promo_sk#44] +Arguments: [cs_promo_sk#29], [p_promo_sk#44], Inner, BuildRight + +(64) CometProject +Input [7]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, p_promo_sk#44] +Arguments: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(65) CometHashAggregate +Input [5]: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(66) CometExchange +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Arguments: hashpartitioning(cp_catalog_page_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(67) CometHashAggregate +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(68) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#56)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(69) CometFilter +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Condition : ((isnotnull(ws_web_site_sk#51) AND isnotnull(ws_item_sk#50)) AND isnotnull(ws_promo_sk#52)) + +(70) CometExchange +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: hashpartitioning(ws_item_sk#50, ws_order_number#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(71) CometSort +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56], [ws_item_sk#50 ASC NULLS FIRST, ws_order_number#53 ASC NULLS FIRST] + +(72) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] +ReadSchema: struct + +(73) CometFilter +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Condition : (isnotnull(wr_item_sk#57) AND isnotnull(wr_order_number#58)) + +(74) CometProject +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] + +(75) CometExchange +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: hashpartitioning(wr_item_sk#57, wr_order_number#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(76) CometSort +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57 ASC NULLS FIRST, wr_order_number#58 ASC NULLS FIRST] + +(77) CometSortMergeJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Right output [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_order_number#53], [wr_item_sk#57, wr_order_number#58], LeftOuter + +(78) CometProject +Input [11]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] + +(79) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#62] + +(80) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] +Right output [1]: [d_date_sk#62] +Arguments: [ws_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(81) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60, d_date_sk#62] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] + +(82) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#63, web_site_id#64] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(83) CometFilter +Input [2]: [web_site_sk#63, web_site_id#64] +Condition : isnotnull(web_site_sk#63) + +(84) CometProject +Input [2]: [web_site_sk#63, web_site_id#64] +Arguments: [web_site_sk#63, web_site_id#65], [web_site_sk#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#64, 16, true, false, true) AS web_site_id#65] + +(85) CometBroadcastExchange +Input [2]: [web_site_sk#63, web_site_id#65] +Arguments: [web_site_sk#63, web_site_id#65] + +(86) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] +Right output [2]: [web_site_sk#63, web_site_id#65] +Arguments: [ws_web_site_sk#51], [web_site_sk#63], Inner, BuildRight + +(87) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_sk#63, web_site_id#65] +Arguments: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(88) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#66] + +(89) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [i_item_sk#66] +Arguments: [ws_item_sk#50], [i_item_sk#66], Inner, BuildRight + +(90) CometProject +Input [8]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, i_item_sk#66] +Arguments: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(91) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#67] + +(92) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [p_promo_sk#67] +Arguments: [ws_promo_sk#52], [p_promo_sk#67], Inner, BuildRight + +(93) CometProject +Input [7]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, p_promo_sk#67] +Arguments: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(94) CometHashAggregate +Input [5]: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Keys [1]: [web_site_id#65] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#54)), partial_sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(95) CometExchange +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Arguments: hashpartitioning(web_site_id#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(96) CometHashAggregate +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Keys [1]: [web_site_id#65] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#54)), sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(97) CometUnion +Child 0 Input [5]: [sales#73, returns#74, profit#75, channel#76, id#77] +Child 1 Input [5]: [sales#78, returns#79, profit#80, channel#81, id#82] +Child 2 Input [5]: [sales#83, returns#84, profit#85, channel#86, id#87] + +(98) CometExpand +Input [5]: [sales#73, returns#74, profit#75, channel#76, id#77] +Arguments: [[sales#73, returns#74, profit#75, channel#76, id#77, 0], [sales#73, returns#74, profit#75, channel#76, null, 1], [sales#73, returns#74, profit#75, null, null, 3]], [sales#73, returns#74, profit#75, channel#88, id#89, spark_grouping_id#90] + +(99) CometHashAggregate +Input [6]: [sales#73, returns#74, profit#75, channel#88, id#89, spark_grouping_id#90] +Keys [3]: [channel#88, id#89, spark_grouping_id#90] +Functions [3]: [partial_sum(sales#73), partial_sum(returns#74), partial_sum(profit#75)] + +(100) CometExchange +Input [9]: [channel#88, id#89, spark_grouping_id#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(channel#88, id#89, spark_grouping_id#90, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(101) CometHashAggregate +Input [9]: [channel#88, id#89, spark_grouping_id#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Keys [3]: [channel#88, id#89, spark_grouping_id#90] +Functions [3]: [sum(sales#73), sum(returns#74), sum(profit#75)] + +(102) CometTakeOrderedAndProject +Input [5]: [channel#88, id#89, sales#97, returns#98, profit#99] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#88 ASC NULLS FIRST,id#89 ASC NULLS FIRST], output=[channel#88,id#89,sales#97,returns#98,profit#99]), [channel#88, id#89, sales#97, returns#98, profit#99], 100, 0, [channel#88 ASC NULLS FIRST, id#89 ASC NULLS FIRST], [channel#88, id#89, sales#97, returns#98, profit#99] + +(103) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#88, id#89, sales#97, returns#98, profit#99] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/extended.txt new file mode 100644 index 0000000000..dd8065cb77 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/extended.txt @@ -0,0 +1,123 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_site + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 120 out of 120 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/simplified.txt index b4a958104e..4a2e5b04ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/simplified.txt @@ -1,81 +1,105 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #4 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #7 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #8 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #9 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #10 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #4 - ReusedExchange [d_date_sk] #5 - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 - ReusedExchange [i_item_sk] #7 - ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [web_site_id,sum,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id,spark_grouping_id] #1 + CometHashAggregate [sales,returns,profit] [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometExpand [channel,id] [sales,returns,profit,channel,id,spark_grouping_id] + CometUnion [sales,returns,profit,channel,id] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),s_store_id] + CometExchange [s_store_id] #2 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #4 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #8 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),cp_catalog_page_id] + CometExchange [cp_catalog_page_id] #9 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #10 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #11 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),web_site_id] + CometExchange [web_site_id] #13 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #14 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #15 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/explain.txt index 9e65a13766..b2f3760446 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/explain.txt @@ -1,64 +1,74 @@ == Physical Plan == -* ColumnarToRow (47) -+- CometTakeOrderedAndProject (46) - +- CometProject (45) - +- CometBroadcastHashJoin (44) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometFilter (17) - : : : +- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- CometBroadcastExchange (33) - : : +- CometFilter (32) - : : +- CometHashAggregate (31) - : : +- CometExchange (30) - : : +- CometHashAggregate (29) - : : +- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (18) - : : : +- ReusedExchange (20) - : : +- ReusedExchange (23) - : +- CometBroadcastExchange (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (36) - +- CometBroadcastExchange (43) - +- CometFilter (42) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (41) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometNativeScan parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#4)] +PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_year#6] @@ -81,168 +91,193 @@ Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#7, ca_state#8] +Right output [2]: [ca_address_sk#7, ca_state#9] Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] +(14) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] -(14) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -(15) CometExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#9] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#9] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -(17) CometFilter -Input [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) - -(18) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Arguments: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(19) CometFilter -Input [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Condition : isnotnull(cr_returning_addr_sk#14) +(19) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#17)] +PushedFilters: [IsNotNull(cr_returning_addr_sk)] +ReadSchema: struct -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +(20) CometFilter +Input [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] +Condition : isnotnull(cr_returning_addr_sk#15) -(21) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [cr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] -(22) CometProject -Input [5]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16, d_date_sk#17] -Arguments: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15], [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15] +(22) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [cr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight -(23) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#18, ca_state#19] +(23) CometProject +Input [5]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17, d_date_sk#18] +Arguments: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16], [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16] -(24) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15] -Right output [2]: [ca_address_sk#18, ca_state#19] -Arguments: [cr_returning_addr_sk#14], [ca_address_sk#18], Inner, BuildRight +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#19, ca_state#9] -(25) CometProject -Input [5]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, ca_address_sk#18, ca_state#19] -Arguments: [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19], [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19] +(25) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16] +Right output [2]: [ca_address_sk#19, ca_state#9] +Arguments: [cr_returning_addr_sk#15], [ca_address_sk#19], Inner, BuildRight -(26) CometHashAggregate -Input [3]: [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19] -Keys [2]: [cr_returning_customer_sk#13, ca_state#19] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#15))] +(26) CometProject +Input [5]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, ca_address_sk#19, ca_state#9] +Arguments: [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9], [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9] -(27) CometExchange -Input [3]: [cr_returning_customer_sk#13, ca_state#19, sum#20] -Arguments: hashpartitioning(cr_returning_customer_sk#13, ca_state#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometHashAggregate +Input [3]: [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9] +Keys [2]: [cr_returning_customer_sk#14, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#16))] -(28) CometHashAggregate -Input [3]: [cr_returning_customer_sk#13, ca_state#19, sum#20] -Keys [2]: [cr_returning_customer_sk#13, ca_state#19] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#15))] +(28) CometExchange +Input [3]: [cr_returning_customer_sk#14, ca_state#9, sum#20] +Arguments: hashpartitioning(cr_returning_customer_sk#14, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate +Input [3]: [cr_returning_customer_sk#14, ca_state#9, sum#20] +Keys [2]: [cr_returning_customer_sk#14, ca_state#9] +Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#16))] + +(30) CometHashAggregate Input [2]: [ctr_state#21, ctr_total_return#22] Keys [1]: [ctr_state#21] Functions [1]: [partial_avg(ctr_total_return#22)] -(30) CometExchange +(31) CometExchange Input [3]: [ctr_state#21, sum#23, count#24] Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(31) CometHashAggregate +(32) CometHashAggregate Input [3]: [ctr_state#21, sum#23, count#24] Keys [1]: [ctr_state#21] Functions [1]: [avg(ctr_total_return#22)] -(32) CometFilter +(33) CometFilter Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) -(33) CometBroadcastExchange +(34) CometBroadcastExchange Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -(34) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_state#11], [ctr_state#21], Inner, (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight +Arguments: [ctr_state#12], [ctr_state#21], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight -(35) CometProject -Input [5]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_customer_sk#10, ctr_total_return#12], [ctr_customer_sk#10, ctr_total_return#12] +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] -(36) CometNativeScan: `spark_catalog`.`default`.`customer` +(37) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct -(37) CometFilter +(38) CometFilter Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) -(38) CometBroadcastExchange +(39) CometProject Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] - -(39) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#10, ctr_total_return#12] -Right output [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [ctr_customer_sk#10], [c_customer_sk#26], Inner, BuildRight +Arguments: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#27, 16, true, false, true) AS c_customer_id#32, c_current_addr_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#29, 10, true, false, true) AS c_salutation#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#35] -(40) CometProject -Input [8]: [ctr_customer_sk#10, ctr_total_return#12, c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31], [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] +(40) CometBroadcastExchange +Input [6]: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] -(41) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [6]: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [ctr_customer_sk#11], [c_customer_sk#26], Inner, BuildRight -(42) CometFilter -Input [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Condition : ((isnotnull(ca_state#39) AND (ca_state#39 = GA)) AND isnotnull(ca_address_sk#32)) +(42) CometProject +Input [8]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35], [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] -(43) CometBroadcastExchange -Input [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] +(43) CometNativeScan parquet spark_catalog.default.customer_address +Output [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(44) CometBroadcastHashJoin -Left output [6]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Right output [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [c_current_addr_sk#28], [ca_address_sk#32], Inner, BuildRight +(44) CometFilter +Input [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#43, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#36)) (45) CometProject -Input [18]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] +Input [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] +Arguments: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53], [ca_address_sk#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#37, 10, true, false, true) AS ca_street_number#48, ca_street_name#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#39, 15, true, false, true) AS ca_street_type#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#40, 10, true, false, true) AS ca_suite_number#50, ca_city#41, ca_county#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#43, 2, true, false, true) AS ca_state#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#44, 10, true, false, true) AS ca_zip#52, ca_country#45, ca_gmt_offset#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#47, 20, true, false, true) AS ca_location_type#53] + +(46) CometBroadcastExchange +Input [12]: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] + +(47) CometBroadcastHashJoin +Left output [6]: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Right output [12]: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [c_current_addr_sk#28], [ca_address_sk#36], Inner, BuildRight + +(48) CometProject +Input [18]: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35, ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13], [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] -(46) CometTakeOrderedAndProject -Input [16]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#27 ASC NULLS FIRST,c_salutation#29 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,c_last_name#31 ASC NULLS FIRST,ca_street_number#33 ASC NULLS FIRST,ca_street_name#34 ASC NULLS FIRST,ca_street_type#35 ASC NULLS FIRST,ca_suite_number#36 ASC NULLS FIRST,ca_city#37 ASC NULLS FIRST,ca_county#38 ASC NULLS FIRST,ca_state#39 ASC NULLS FIRST,ca_zip#40 ASC NULLS FIRST,ca_country#41 ASC NULLS FIRST,ca_gmt_offset#42 ASC NULLS FIRST,ca_location_type#43 ASC NULLS FIRST,ctr_total_return#12 ASC NULLS FIRST], output=[c_customer_id#27,c_salutation#29,c_first_name#30,c_last_name#31,ca_street_number#33,ca_street_name#34,ca_street_type#35,ca_suite_number#36,ca_city#37,ca_county#38,ca_state#39,ca_zip#40,ca_country#41,ca_gmt_offset#42,ca_location_type#43,ctr_total_return#12]), [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12], 100, [c_customer_id#27 ASC NULLS FIRST, c_salutation#29 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, c_last_name#31 ASC NULLS FIRST, ca_street_number#33 ASC NULLS FIRST, ca_street_name#34 ASC NULLS FIRST, ca_street_type#35 ASC NULLS FIRST, ca_suite_number#36 ASC NULLS FIRST, ca_city#37 ASC NULLS FIRST, ca_county#38 ASC NULLS FIRST, ca_state#39 ASC NULLS FIRST, ca_zip#40 ASC NULLS FIRST, ca_country#41 ASC NULLS FIRST, ca_gmt_offset#42 ASC NULLS FIRST, ca_location_type#43 ASC NULLS FIRST, ctr_total_return#12 ASC NULLS FIRST], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] +(49) CometTakeOrderedAndProject +Input [16]: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#32 ASC NULLS FIRST,c_salutation#33 ASC NULLS FIRST,c_first_name#34 ASC NULLS FIRST,c_last_name#35 ASC NULLS FIRST,ca_street_number#48 ASC NULLS FIRST,ca_street_name#38 ASC NULLS FIRST,ca_street_type#49 ASC NULLS FIRST,ca_suite_number#50 ASC NULLS FIRST,ca_city#41 ASC NULLS FIRST,ca_county#42 ASC NULLS FIRST,ca_state#51 ASC NULLS FIRST,ca_zip#52 ASC NULLS FIRST,ca_country#45 ASC NULLS FIRST,ca_gmt_offset#46 ASC NULLS FIRST,ca_location_type#53 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#32,c_salutation#33,c_first_name#34,c_last_name#35,ca_street_number#48,ca_street_name#38,ca_street_type#49,ca_suite_number#50,ca_city#41,ca_county#42,ca_state#51,ca_zip#52,ca_country#45,ca_gmt_offset#46,ca_location_type#53,ctr_total_return#13]), [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13], 100, 0, [c_customer_id#32 ASC NULLS FIRST, c_salutation#33 ASC NULLS FIRST, c_first_name#34 ASC NULLS FIRST, c_last_name#35 ASC NULLS FIRST, ca_street_number#48 ASC NULLS FIRST, ca_street_name#38 ASC NULLS FIRST, ca_street_type#49 ASC NULLS FIRST, ca_suite_number#50 ASC NULLS FIRST, ca_city#41 ASC NULLS FIRST, ca_county#42 ASC NULLS FIRST, ca_state#51 ASC NULLS FIRST, ca_zip#52 ASC NULLS FIRST, ca_country#45 ASC NULLS FIRST, ca_gmt_offset#46 ASC NULLS FIRST, ca_location_type#53 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] -(47) ColumnarToRow [codegen id : 1] -Input [16]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] +(50) CometColumnarToRow [codegen id : 1] +Input [16]: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/extended.txt new file mode 100644 index 0000000000..188bc4f72b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/extended.txt @@ -0,0 +1,58 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 55 out of 55 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/simplified.txt index 807434f4a6..a4d33bfd0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] @@ -9,41 +9,44 @@ WholeStageCodegen (1) CometProject [ctr_customer_sk,ctr_total_return] CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] CometExchange [cr_returning_customer_sk,ca_state] #1 - CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return)] CometExchange [ctr_state] #5 - CometHashAggregate [ctr_state,sum,count,ctr_total_return] - CometHashAggregate [ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [cr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] CometExchange [cr_returning_customer_sk,ca_state] #6 - CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] ReusedExchange [d_date_sk] #2 ReusedExchange [ca_address_sk,ca_state] #3 CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #7 - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #8 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/explain.txt index fdcea86558..f63414fbe4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (26) +* CometColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) +- CometExchange (23) @@ -13,23 +13,26 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) + : : +- CometNativeScan parquet spark_catalog.default.inventory (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) +- CometProject (19) +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (17) + +- CometNativeScan parquet spark_catalog.default.store_sales (17) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,423,821]), IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] @@ -37,101 +40,111 @@ Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) A (3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#9)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] +ReadSchema: struct (5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) (6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] (7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] (8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight (9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +(10) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] +ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11, ss_sold_date_sk#12] +(17) CometNativeScan parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (18) CometFilter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) (19) CometProject -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11], [ss_item_sk#11] +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] (20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#11] -Arguments: [i_item_sk#1], [ss_item_sk#11], Inner, BuildLeft +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/extended.txt new file mode 100644 index 0000000000..6e00d25e4c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.inventory + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_sales + +Comet accelerated 25 out of 25 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/simplified.txt index b8535baaf2..4deaf58d7e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] @@ -12,17 +12,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [ss_item_sk] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/explain.txt index e02d453853..193387d80a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/explain.txt @@ -1,184 +1,316 @@ == Physical Plan == -* ColumnarToRow (35) -+- CometTakeOrderedAndProject (34) - +- CometProject (33) - +- CometBroadcastHashJoin (32) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometHashAggregate (25) - : : +- CometExchange (24) - : : +- CometHashAggregate (23) - : : +- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (56) ++- CometTakeOrderedAndProject (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometFilter (12) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) - : +- CometBroadcastExchange (28) - : +- CometHashAggregate (27) - : +- ReusedExchange (26) - +- ReusedExchange (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometNativeScan parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (38) + : +- CometHashAggregate (37) + : +- CometExchange (36) + : +- CometHashAggregate (35) + : +- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- ReusedExchange (32) + +- CometBroadcastExchange (52) + +- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) + + +(1) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Arguments: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#3)] +PushedFilters: [IsNotNull(sr_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] + +(7) CometBroadcastHashJoin Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] +Right output [2]: [i_item_sk#4, i_item_id#6] Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6, d_date#7] - -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8, d_week_seq#9] - -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#10, d_week_seq#11] -Arguments: [d_date#10, d_week_seq#11] - -(12) CometFilter -Input [2]: [d_date#10, d_week_seq#11] -Condition : cast(d_date#10 as string) IN (2000-06-30,2000-09-27,2000-11-17) - -(13) CometProject -Input [2]: [d_date#10, d_week_seq#11] -Arguments: [d_week_seq#11], [d_week_seq#11] - -(14) CometBroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: [d_week_seq#11] - -(15) CometBroadcastHashJoin -Left output [2]: [d_date#8, d_week_seq#9] -Right output [1]: [d_week_seq#11] -Arguments: [d_week_seq#9], [d_week_seq#11], LeftSemi, BuildRight - -(16) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] - -(17) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] - -(18) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight - -(19) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(21) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#3], [d_date_sk#6], Inner, BuildRight - -(22) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [sr_return_quantity#2, i_item_id#5], [sr_return_quantity#2, i_item_id#5] - -(23) CometHashAggregate -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(8) CometProject +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) + +(11) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(12) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#11, d_week_seq#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date#11, d_week_seq#12] +Condition : cast(d_date#11 as string) IN (2000-06-30,2000-09-27,2000-11-17) + +(14) CometProject +Input [2]: [d_date#11, d_week_seq#12] +Arguments: [d_week_seq#12], [d_week_seq#12] + +(15) CometBroadcastExchange +Input [1]: [d_week_seq#12] +Arguments: [d_week_seq#12] + +(16) CometBroadcastHashJoin +Left output [2]: [d_date#9, d_week_seq#10] +Right output [1]: [d_week_seq#12] +Arguments: [d_week_seq#10], [d_week_seq#12], LeftSemi, BuildRight + +(17) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] + +(18) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] + +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight + +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -(24) CometExchange -Input [2]: [i_item_id#5, sum#12] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) CometHashAggregate -Input [2]: [i_item_id#5, sum#12] -Keys [1]: [i_item_id#5] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -(26) ReusedExchange [Reuses operator id: 24] -Output [2]: [i_item_id#13, sum#14] - -(27) CometHashAggregate -Input [2]: [i_item_id#13, sum#14] -Keys [1]: [i_item_id#13] +(27) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#16)] +PushedFilters: [IsNotNull(cr_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Condition : isnotnull(cr_item_sk#14) + +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#17, i_item_id#18] + +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Right output [2]: [i_item_sk#17, i_item_id#18] +Arguments: [cr_item_sk#14], [i_item_sk#17], Inner, BuildRight + +(31) CometProject +Input [5]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16, i_item_sk#17, i_item_id#18] +Arguments: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18], [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] + +(32) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#19] + +(33) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#16], [d_date_sk#19], Inner, BuildRight + +(34) CometProject +Input [4]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18, d_date_sk#19] +Arguments: [cr_return_quantity#15, i_item_id#18], [cr_return_quantity#15, i_item_id#18] + +(35) CometHashAggregate +Input [2]: [cr_return_quantity#15, i_item_id#18] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(cr_return_quantity#15)] + +(36) CometExchange +Input [2]: [i_item_id#18, sum#20] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(37) CometHashAggregate +Input [2]: [i_item_id#18, sum#20] +Keys [1]: [i_item_id#18] Functions [1]: [sum(cr_return_quantity#15)] -(28) CometBroadcastExchange -Input [2]: [item_id#16, cr_item_qty#17] -Arguments: [item_id#16, cr_item_qty#17] - -(29) CometBroadcastHashJoin -Left output [2]: [item_id#18, sr_item_qty#19] -Right output [2]: [item_id#16, cr_item_qty#17] -Arguments: [item_id#18], [item_id#16], Inner, BuildRight - -(30) CometProject -Input [4]: [item_id#18, sr_item_qty#19, item_id#16, cr_item_qty#17] -Arguments: [item_id#18, sr_item_qty#19, cr_item_qty#17], [item_id#18, sr_item_qty#19, cr_item_qty#17] - -(31) ReusedExchange [Reuses operator id: 28] -Output [2]: [item_id#20, wr_item_qty#21] - -(32) CometBroadcastHashJoin -Left output [3]: [item_id#18, sr_item_qty#19, cr_item_qty#17] -Right output [2]: [item_id#20, wr_item_qty#21] -Arguments: [item_id#18], [item_id#20], Inner, BuildRight - -(33) CometProject -Input [5]: [item_id#18, sr_item_qty#19, cr_item_qty#17, item_id#20, wr_item_qty#21] -Arguments: [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25], [item_id#18, sr_item_qty#19, (((cast(sr_item_qty#19 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as double)))) / 3.0) * 100.0) AS sr_dev#22, cr_item_qty#17, (((cast(cr_item_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as double)))) / 3.0) * 100.0) AS cr_dev#23, wr_item_qty#21, (((cast(wr_item_qty#21 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as double)))) / 3.0) * 100.0) AS wr_dev#24, (cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as decimal(20,0)) / 3.0) AS average#25] - -(34) CometTakeOrderedAndProject -Input [8]: [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#18 ASC NULLS FIRST,sr_item_qty#19 ASC NULLS FIRST], output=[item_id#18,sr_item_qty#19,sr_dev#22,cr_item_qty#17,cr_dev#23,wr_item_qty#21,wr_dev#24,average#25]), [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25], 100, [item_id#18 ASC NULLS FIRST, sr_item_qty#19 ASC NULLS FIRST], [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25] - -(35) ColumnarToRow [codegen id : 1] -Input [8]: [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25] +(38) CometBroadcastExchange +Input [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#21, cr_item_qty#22] + +(39) CometBroadcastHashJoin +Left output [2]: [item_id#23, sr_item_qty#24] +Right output [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#23], [item_id#21], Inner, BuildRight + +(40) CometProject +Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] +Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] + +(41) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#27)] +PushedFilters: [IsNotNull(wr_item_sk)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Condition : isnotnull(wr_item_sk#25) + +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#28, i_item_id#29] + +(44) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Right output [2]: [i_item_sk#28, i_item_id#29] +Arguments: [wr_item_sk#25], [i_item_sk#28], Inner, BuildRight + +(45) CometProject +Input [5]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29], [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] + +(46) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#30] + +(47) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] +Right output [1]: [d_date_sk#30] +Arguments: [wr_returned_date_sk#27], [d_date_sk#30], Inner, BuildRight + +(48) CometProject +Input [4]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [wr_return_quantity#26, i_item_id#29], [wr_return_quantity#26, i_item_id#29] + +(49) CometHashAggregate +Input [2]: [wr_return_quantity#26, i_item_id#29] +Keys [1]: [i_item_id#29] +Functions [1]: [partial_sum(wr_return_quantity#26)] + +(50) CometExchange +Input [2]: [i_item_id#29, sum#31] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(51) CometHashAggregate +Input [2]: [i_item_id#29, sum#31] +Keys [1]: [i_item_id#29] +Functions [1]: [sum(wr_return_quantity#26)] + +(52) CometBroadcastExchange +Input [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#32, wr_item_qty#33] + +(53) CometBroadcastHashJoin +Left output [3]: [item_id#23, sr_item_qty#24, cr_item_qty#22] +Right output [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#23], [item_id#32], Inner, BuildRight + +(54) CometProject +Input [5]: [item_id#23, sr_item_qty#24, cr_item_qty#22, item_id#32, wr_item_qty#33] +Arguments: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#23, sr_item_qty#24, (((cast(sr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] + +(55) CometTakeOrderedAndProject +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#23 ASC NULLS FIRST,sr_item_qty#24 ASC NULLS FIRST], output=[item_id#23,sr_item_qty#24,sr_dev#34,cr_item_qty#22,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#23 ASC NULLS FIRST, sr_item_qty#24 ASC NULLS FIRST], [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] + +(56) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/extended.txt new file mode 100644 index 0000000000..742a696f6f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/extended.txt @@ -0,0 +1,88 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 85 out of 85 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/simplified.txt index f8447c7147..b787401379 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/simplified.txt @@ -1,37 +1,58 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] CometProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] CometBroadcastHashJoin [item_id,sr_item_qty,cr_item_qty,item_id,wr_item_qty] CometProject [item_id,sr_item_qty,cr_item_qty] CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] - CometHashAggregate [item_id,sr_item_qty,i_item_id,sum,sum(sr_return_quantity)] + CometHashAggregate [sum] [item_id,sr_item_qty,i_item_id,sum(sr_return_quantity)] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,sr_return_quantity] + CometHashAggregate [sr_return_quantity] [i_item_id,sum] CometProject [sr_return_quantity,i_item_id] CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #4 CometProject [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [d_week_seq] #5 CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [item_id,cr_item_qty] #6 - CometHashAggregate [item_id,cr_item_qty,i_item_id,sum,sum(cr_return_quantity)] - ReusedExchange [i_item_id,sum] #1 - ReusedExchange [item_id,wr_item_qty] #6 + CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] + CometExchange [i_item_id] #7 + CometHashAggregate [cr_return_quantity] [i_item_id,sum] + CometProject [cr_return_quantity,i_item_id] + CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] + CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [item_id,wr_item_qty] #8 + CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] + CometExchange [i_item_id] #9 + CometHashAggregate [wr_return_quantity] [i_item_id,sum] + CometProject [wr_return_quantity,i_item_id] + CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] + CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/explain.txt index 37a2d3f738..63140cfc2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/explain.txt @@ -1,167 +1,190 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * ColumnarToRow (30) - +- CometBroadcastHashJoin (29) - :- CometBroadcastExchange (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (9) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (14) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`income_band` (19) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (10) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.income_band (20) + +- CometProject (29) + +- CometFilter (28) + +- CometNativeScan parquet spark_catalog.default.store_returns (27) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7, ca_city#8] - -(4) CometFilter -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) - -(5) CometProject -Input [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7], [ca_address_sk#7] - -(6) CometBroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: [ca_address_sk#7] - -(7) CometBroadcastHashJoin -Left output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Right output [1]: [ca_address_sk#7] -Arguments: [c_current_addr_sk#4], [ca_address_sk#7], Inner, BuildRight - -(8) CometProject -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6], [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(10) CometFilter -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) - -(11) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(12) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Right output [1]: [cd_demo_sk#9] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(13) CometProject -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(15) CometFilter -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) - -(16) CometBroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#10], Inner, BuildRight - -(18) CometProject -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] - -(19) CometNativeScan: `spark_catalog`.`default`.`income_band` -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(20) CometFilter -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) - -(21) CometProject -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12], [ib_income_band_sk#12] - -(22) CometBroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: [ib_income_band_sk#12] - -(23) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Right output [1]: [ib_income_band_sk#12] -Arguments: [hd_income_band_sk#11], [ib_income_band_sk#12], Inner, BuildRight - -(24) CometProject -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(25) CometBroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15, sr_returned_date_sk#16] - -(27) CometFilter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) - -(28) CometProject -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15], [sr_cdemo_sk#15] - -(29) CometBroadcastHashJoin -Left output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [1]: [sr_cdemo_sk#15] -Arguments: [cd_demo_sk#9], [sr_cdemo_sk#15], Inner, BuildLeft - -(30) ColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(31) Project [codegen id : 1] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(32) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] +(3) CometProject +Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#1, 16, true, false, true) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#5, 20, true, false, true) AS c_first_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#6, 30, true, false, true) AS c_last_name#9] + +(4) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#10, ca_city#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [ca_address_sk#10, ca_city#11] +Condition : ((isnotnull(ca_city#11) AND (ca_city#11 = Edgewood)) AND isnotnull(ca_address_sk#10)) + +(6) CometProject +Input [2]: [ca_address_sk#10, ca_city#11] +Arguments: [ca_address_sk#10], [ca_address_sk#10] + +(7) CometBroadcastExchange +Input [1]: [ca_address_sk#10] +Arguments: [ca_address_sk#10] + +(8) CometBroadcastHashJoin +Left output [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9] +Right output [1]: [ca_address_sk#10] +Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight + +(9) CometProject +Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] + +(10) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(11) CometFilter +Input [1]: [cd_demo_sk#12] +Condition : isnotnull(cd_demo_sk#12) + +(12) CometBroadcastExchange +Input [1]: [cd_demo_sk#12] +Arguments: [cd_demo_sk#12] + +(13) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] +Right output [1]: [cd_demo_sk#12] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight + +(14) CometProject +Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(15) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Condition : (isnotnull(hd_demo_sk#13) AND isnotnull(hd_income_band_sk#14)) + +(17) CometBroadcastExchange +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [hd_demo_sk#13, hd_income_band_sk#14] + +(18) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight + +(19) CometProject +Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] + +(20) CometNativeScan parquet spark_catalog.default.income_band +Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Condition : ((((isnotnull(ib_lower_bound#16) AND isnotnull(ib_upper_bound#17)) AND (ib_lower_bound#16 >= 38128)) AND (ib_upper_bound#17 <= 88128)) AND isnotnull(ib_income_band_sk#15)) + +(22) CometProject +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Arguments: [ib_income_band_sk#15], [ib_income_band_sk#15] + +(23) CometBroadcastExchange +Input [1]: [ib_income_band_sk#15] +Arguments: [ib_income_band_sk#15] + +(24) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] +Right output [1]: [ib_income_band_sk#15] +Arguments: [hd_income_band_sk#14], [ib_income_band_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14, ib_income_band_sk#15] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(26) CometBroadcastExchange +Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(27) CometNativeScan parquet spark_catalog.default.store_returns +Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_cdemo_sk)] +ReadSchema: struct + +(28) CometFilter +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_cdemo_sk#18) + +(29) CometProject +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Arguments: [sr_cdemo_sk#18], [sr_cdemo_sk#18] + +(30) CometBroadcastHashJoin +Left output [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [1]: [sr_cdemo_sk#18] +Arguments: [cd_demo_sk#12], [sr_cdemo_sk#18], Inner, BuildLeft + +(31) CometProject +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] +Arguments: [customer_id#20, customername#21, c_customer_id#7], [c_customer_id#7 AS customer_id#20, concat(c_last_name#9, , , c_first_name#8) AS customername#21, c_customer_id#7] + +(32) CometTakeOrderedAndProject +Input [3]: [customer_id#20, customername#21, c_customer_id#7] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#7 ASC NULLS FIRST], output=[customer_id#20,customername#21]), [customer_id#20, customername#21], 100, 0, [c_customer_id#7 ASC NULLS FIRST], [customer_id#20, customername#21] + +(33) CometColumnarToRow [codegen id : 1] +Input [2]: [customer_id#20, customername#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/extended.txt new file mode 100644 index 0000000000..b025c868d1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_returns + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/simplified.txt index 6fd5331780..2f4baa9f3f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ -TakeOrderedAndProject [c_customer_id,customer_id,customername] - WholeStageCodegen (1) - Project [c_customer_id,c_last_name,c_first_name] - ColumnarToRow - InputAdapter +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customername,c_customer_id] + CometProject [c_last_name,c_first_name] [customer_id,customername,c_customer_id] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk] @@ -13,22 +13,23 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] - CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometNativeScan: `spark_catalog`.`default`.`income_band` [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_cdemo_sk,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/explain.txt index 5a77d6f24d..d4ba3e2a08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/explain.txt @@ -1,54 +1,61 @@ == Physical Plan == -* ColumnarToRow (45) -+- CometTakeOrderedAndProject (44) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) +* CometColumnarToRow (48) ++- CometTakeOrderedAndProject (47) + +- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometBroadcastHashJoin (42) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) : : : : :- CometProject (13) : : : : : +- CometBroadcastHashJoin (12) : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometBroadcastExchange (3) : : : : : : : +- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (9) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometFilter (15) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (14) - : : : +- CometBroadcastExchange (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (19) - : : +- CometBroadcastExchange (27) - : : +- CometProject (26) - : : +- CometFilter (25) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - : +- CometBroadcastExchange (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometProject (16) + : : : : +- CometFilter (15) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (14) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.date_dim (32) + +- CometBroadcastExchange (41) + +- CometProject (40) + +- CometFilter (39) + +- CometNativeScan parquet spark_catalog.default.reason (38) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] +ReadSchema: struct (2) CometFilter Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] @@ -58,9 +65,12 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(4) CometNativeScan parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] +ReadSchema: struct (5) CometFilter Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] @@ -79,9 +89,12 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(9) CometNativeScan: `spark_catalog`.`default`.`web_page` +(9) CometNativeScan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#17] -Arguments: [wp_web_page_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct (10) CometFilter Input [1]: [wp_web_page_sk#17] @@ -100,137 +113,164 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(14) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(14) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (15) CometFilter Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(16) CometBroadcastExchange +(16) CometProject Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(17) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight - -(18) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(20) CometFilter -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) +(17) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(21) CometBroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(22) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Right output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20], [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23], Inner, BuildRight - -(23) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(18) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight + +(19) CometProject +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] + +(20) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true))) + +(22) CometProject +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#27] + +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] + +(24) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25, ca_country#26] +(26) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct -(25) CometFilter -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) +(27) CometFilter +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (LA,IA,AR))) -(26) CometProject -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25], [ca_address_sk#24, ca_state#25] +(28) CometProject +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) AS ca_state#31] -(27) CometBroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: [ca_address_sk#24, ca_state#25] +(29) CometBroadcastExchange +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] -(28) CometBroadcastHashJoin +(30) CometBroadcastHashJoin Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [ca_address_sk#24, ca_state#25] -Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#24], Inner, ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight -(29) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] +(31) CometProject +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27, d_year#28] +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#32, d_year#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct -(31) CometFilter -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +(33) CometFilter +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) -(32) CometProject -Input [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27], [d_date_sk#27] +(34) CometProject +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] -(33) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] +(35) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(34) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [1]: [d_date_sk#27] -Arguments: [ws_sold_date_sk#7], [d_date_sk#27], Inner, BuildRight +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight -(35) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] +(37) CometProject +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(36) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] +(38) CometNativeScan parquet spark_catalog.default.reason +Output [2]: [r_reason_sk#34, r_reason_desc#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_sk)] +ReadSchema: struct -(37) CometFilter -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) +(39) CometFilter +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) -(38) CometBroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] +(40) CometProject +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#35, 100, true, false, true) AS r_reason_desc#36] + +(41) CometBroadcastExchange +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] -(39) CometBroadcastHashJoin +(42) CometBroadcastHashJoin Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [wr_reason_sk#12], [r_reason_sk#29], Inner, BuildRight +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight -(40) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] -Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] +(43) CometProject +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] -(41) CometHashAggregate -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] +(44) CometHashAggregate +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] -(42) CometExchange -Input [7]: [r_reason_desc#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(45) CometExchange +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(43) CometHashAggregate -Input [7]: [r_reason_desc#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Keys [1]: [r_reason_desc#30] +(46) CometHashAggregate +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] -(44) CometTakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#37 ASC NULLS FIRST,avg(ws_quantity)#38 ASC NULLS FIRST,avg(wr_refunded_cash)#39 ASC NULLS FIRST,avg(wr_fee)#40 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#37,avg(ws_quantity)#38,avg(wr_refunded_cash)#39,avg(wr_fee)#40]), [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40], 100, [substr(r_reason_desc, 1, 20)#37 ASC NULLS FIRST, avg(ws_quantity)#38 ASC NULLS FIRST, avg(wr_refunded_cash)#39 ASC NULLS FIRST, avg(wr_fee)#40 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] +(47) CometTakeOrderedAndProject +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] -(45) ColumnarToRow [codegen id : 1] -Input [4]: [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] +(48) CometColumnarToRow [codegen id : 1] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/extended.txt new file mode 100644 index 0000000000..358da39c56 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/extended.txt @@ -0,0 +1,50 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 47 out of 47 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/simplified.txt index 905125e838..925b2ed9b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - CometHashAggregate [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,sum,count,sum,count,sum,count,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] + CometHashAggregate [sum,count,sum,count,sum,count] [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] CometExchange [r_reason_desc] #1 - CometHashAggregate [r_reason_desc,sum,count,sum,count,sum,count,ws_quantity,wr_refunded_cash,wr_fee] + CometHashAggregate [ws_quantity,wr_refunded_cash,wr_fee] [r_reason_desc,sum,count,sum,count,sum,count] CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] @@ -21,27 +21,30 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometBroadcastExchange [wp_web_page_sk] #3 CometFilter [wp_web_page_sk] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #6 - CometProject [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 - CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] + CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] + CometFilter [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/explain.txt index 22e2a3e664..e0b9dfa394 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/explain.txt @@ -1,40 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (23) -+- * Project (22) - +- Window (21) - +- * ColumnarToRow (20) - +- CometSort (19) - +- CometExchange (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometExpand (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometExchange (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometExpand (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- CometBroadcastExchange (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -57,65 +65,72 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_category#8, i_class#7], [ws_net_paid#2, i_category#8, i_class#7] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] -(14) CometExpand -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] +(15) CometExpand +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] -(15) CometHashAggregate -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(16) CometHashAggregate +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(16) CometExchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#12] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(17) CometExchange +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) CometHashAggregate -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#12] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(18) CometHashAggregate +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -(18) CometExchange -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: hashpartitioning(_w1#16, _w2#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(19) CometExchange +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(19) CometSort -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17], [_w1#16 ASC NULLS FIRST, _w2#17 ASC NULLS FIRST, _w0#15 DESC NULLS LAST] +(20) CometSort +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19], [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(20) ColumnarToRow [codegen id : 1] -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] -(21) Window -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: [rank(_w0#15) windowspecdefinition(_w1#16, _w2#17, _w0#15 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#18], [_w1#16, _w2#17], [_w0#15 DESC NULLS LAST] +(22) Window +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] -(22) Project [codegen id : 2] -Output [5]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] -Input [8]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17, rank_within_parent#18] +(23) Project [codegen id : 2] +Output [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Input [8]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] -(23) TakeOrderedAndProject -Input [5]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] -Arguments: 100, [lochierarchy#14 DESC NULLS LAST, CASE WHEN (lochierarchy#14 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#18 ASC NULLS FIRST], [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] +(24) TakeOrderedAndProject +Input [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/extended.txt new file mode 100644 index 0000000000..cf1ba69679 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/extended.txt @@ -0,0 +1,26 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 20 out of 23 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/simplified.txt index 777ca4bdfe..81a2237a29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/simplified.txt @@ -4,24 +4,25 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] CometExchange [_w1,_w2] #1 - CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [sum] [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ws_net_paid)),spark_grouping_id] CometExchange [i_category,i_class,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,spark_grouping_id,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [i_category,i_class,spark_grouping_id,sum] CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] CometProject [ws_net_paid,i_category,i_class] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/explain.txt index 3ff75a4a3b..c721c81f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/explain.txt @@ -1,45 +1,75 @@ == Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin LeftAnti BuildRight (24) - :- * BroadcastHashJoin LeftAnti BuildRight (22) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometHashAggregate (19) - : +- ReusedExchange (18) - +- ReusedExchange (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* HashAggregate (51) ++- * CometColumnarToRow (50) + +- CometColumnarExchange (49) + +- * HashAggregate (48) + +- * Project (47) + +- * BroadcastHashJoin LeftAnti BuildRight (46) + :- * BroadcastHashJoin LeftAnti BuildRight (32) + : :- * CometColumnarToRow (18) + : : +- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (31) + : +- * CometColumnarToRow (30) + : +- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometFilter (20) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (19) + : : +- ReusedExchange (21) + : +- ReusedExchange (24) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : +- ReusedExchange (35) + +- ReusedExchange (38) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] @@ -62,93 +92,199 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(9) CometNativeScan: `spark_catalog`.`default`.`customer` +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(17) ColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(18) CometColumnarToRow [codegen id : 3] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] + +(19) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) + +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(18) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] +(23) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(19) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] +(24) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] + +(25) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight + +(26) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] + +(27) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(20) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] +(28) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(21) BroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=2] +(29) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] +Functions: [] -(22) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)] +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] + +(31) BroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] + +(32) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None -(23) ReusedExchange [Reuses operator id: 21] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(24) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] +(35) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight + +(37) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] + +(38) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] + +(39) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] + +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(42) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(44) CometColumnarToRow [codegen id : 2] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] + +(45) BroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] + +(46) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None -(25) Project [codegen id : 3] +(47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(26) HashAggregate [codegen id : 3] +(48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [1]: [count#16] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] + +(49) CometColumnarExchange +Input [1]: [count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(27) Exchange -Input [1]: [count#16] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [count#26] -(28) HashAggregate [codegen id : 4] -Input [1]: [count#16] +(51) HashAggregate [codegen id : 4] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [1]: [count(1)#17 AS count(1)#18] +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/extended.txt new file mode 100644 index 0000000000..f659e21953 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/extended.txt @@ -0,0 +1,65 @@ +HashAggregate ++- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 52 out of 59 eligible operators (88%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/simplified.txt index 8612445652..0fe68e3efa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/simplified.txt @@ -1,36 +1,61 @@ WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - ColumnarToRow + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #2 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #3 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - InputAdapter - ReusedExchange [c_last_name,c_first_name,d_date] #5 + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/explain.txt index 1917778c34..25b4c305f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/explain.txt @@ -6,7 +6,7 @@ : : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) : : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : : : : : :- * ColumnarToRow (25) +: : : : : : :- * CometColumnarToRow (25) : : : : : : : +- CometHashAggregate (24) : : : : : : : +- CometExchange (23) : : : : : : : +- CometHashAggregate (22) @@ -18,21 +18,21 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (16) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) -: : : : : : +- * ColumnarToRow (44) +: : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) : : : : : : +- CometExchange (42) : : : : : : +- CometHashAggregate (41) @@ -44,15 +44,15 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) -: : : : : +- * ColumnarToRow (65) +: : : : : +- * CometColumnarToRow (65) : : : : : +- CometHashAggregate (64) : : : : : +- CometExchange (63) : : : : : +- CometHashAggregate (62) @@ -64,15 +64,15 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (53) +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) -: : : : +- * ColumnarToRow (86) +: : : : +- * CometColumnarToRow (86) : : : : +- CometHashAggregate (85) : : : : +- CometExchange (84) : : : : +- CometHashAggregate (83) @@ -84,15 +84,15 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (68) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (74) +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) -: : : +- * ColumnarToRow (107) +: : : +- * CometColumnarToRow (107) : : : +- CometHashAggregate (106) : : : +- CometExchange (105) : : : +- CometHashAggregate (104) @@ -104,15 +104,15 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (89) +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (95) +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) -: : +- * ColumnarToRow (128) +: : +- * CometColumnarToRow (128) : : +- CometHashAggregate (127) : : +- CometExchange (126) : : +- CometHashAggregate (125) @@ -124,15 +124,15 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (110) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (116) +: : : +- CometNativeScan parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) -: +- * ColumnarToRow (149) +: +- * CometColumnarToRow (149) : +- CometHashAggregate (148) : +- CometExchange (147) : +- CometHashAggregate (146) @@ -144,15 +144,15 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (131) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (137) +: : +- CometNativeScan parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) - +- * ColumnarToRow (170) + +- * CometColumnarToRow (170) +- CometHashAggregate (169) +- CometExchange (168) +- CometHashAggregate (167) @@ -164,18 +164,21 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (152) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (158) + : +- CometNativeScan parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] @@ -185,9 +188,12 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Arguments: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] @@ -210,9 +216,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Arguments: [t_time_sk#8, t_hour#9, t_minute#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] @@ -235,9 +244,12 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometNativeScan: `spark_catalog`.`default`.`store` +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] -Arguments: [s_store_sk#11, s_store_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [s_store_sk#11, s_store_name#12] @@ -273,12 +285,15 @@ Input [1]: [count#13] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 8] +(25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (27) CometFilter Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] @@ -300,9 +315,12 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Arguments: [t_time_sk#20, t_hour#21, t_minute#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (33) CometFilter Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] @@ -350,7 +368,7 @@ Input [1]: [count#24] Keys: [] Functions [1]: [count(1)] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [h9_to_9_30#25] (45) BroadcastExchange @@ -361,9 +379,12 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(47) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (48) CometFilter Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] @@ -385,9 +406,12 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(53) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (54) CometFilter Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] @@ -435,7 +459,7 @@ Input [1]: [count#35] Keys: [] Functions [1]: [count(1)] -(65) ColumnarToRow [codegen id : 2] +(65) CometColumnarToRow [codegen id : 2] Input [1]: [h9_30_to_10#36] (66) BroadcastExchange @@ -446,9 +470,12 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(68) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (69) CometFilter Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] @@ -470,9 +497,12 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(74) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Arguments: [t_time_sk#42, t_hour#43, t_minute#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (75) CometFilter Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] @@ -520,7 +550,7 @@ Input [1]: [count#46] Keys: [] Functions [1]: [count(1)] -(86) ColumnarToRow [codegen id : 3] +(86) CometColumnarToRow [codegen id : 3] Input [1]: [h10_to_10_30#47] (87) BroadcastExchange @@ -531,9 +561,12 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(89) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (90) CometFilter Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] @@ -555,9 +588,12 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(95) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Arguments: [t_time_sk#53, t_hour#54, t_minute#55] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (96) CometFilter Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] @@ -605,7 +641,7 @@ Input [1]: [count#57] Keys: [] Functions [1]: [count(1)] -(107) ColumnarToRow [codegen id : 4] +(107) CometColumnarToRow [codegen id : 4] Input [1]: [h10_30_to_11#58] (108) BroadcastExchange @@ -616,9 +652,12 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(110) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (111) CometFilter Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] @@ -640,9 +679,12 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(116) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Arguments: [t_time_sk#64, t_hour#65, t_minute#66] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (117) CometFilter Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] @@ -690,7 +732,7 @@ Input [1]: [count#68] Keys: [] Functions [1]: [count(1)] -(128) ColumnarToRow [codegen id : 5] +(128) CometColumnarToRow [codegen id : 5] Input [1]: [h11_to_11_30#69] (129) BroadcastExchange @@ -701,9 +743,12 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(131) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (132) CometFilter Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] @@ -725,9 +770,12 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(137) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Arguments: [t_time_sk#75, t_hour#76, t_minute#77] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (138) CometFilter Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] @@ -775,7 +823,7 @@ Input [1]: [count#79] Keys: [] Functions [1]: [count(1)] -(149) ColumnarToRow [codegen id : 6] +(149) CometColumnarToRow [codegen id : 6] Input [1]: [h11_30_to_12#80] (150) BroadcastExchange @@ -786,9 +834,12 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(152) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (153) CometFilter Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] @@ -810,9 +861,12 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(158) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Arguments: [t_time_sk#86, t_hour#87, t_minute#88] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (159) CometFilter Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] @@ -860,7 +914,7 @@ Input [1]: [count#90] Keys: [] Functions [1]: [count(1)] -(170) ColumnarToRow [codegen id : 7] +(170) CometColumnarToRow [codegen id : 7] Input [1]: [h12_to_12_30#91] (171) BroadcastExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/extended.txt new file mode 100644 index 0000000000..779053cb5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/extended.txt @@ -0,0 +1,216 @@ +BroadcastNestedLoopJoin +:- BroadcastNestedLoopJoin +: :- BroadcastNestedLoopJoin +: : :- BroadcastNestedLoopJoin +: : : :- BroadcastNestedLoopJoin +: : : : :- BroadcastNestedLoopJoin +: : : : : :- BroadcastNestedLoopJoin +: : : : : : :- CometColumnarToRow +: : : : : : : +- CometHashAggregate +: : : : : : : +- CometExchange +: : : : : : : +- CometHashAggregate +: : : : : : : +- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : :- CometProject +: : : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : : :- CometProject +: : : : : : : : : : +- CometFilter +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : : : +- CometBroadcastExchange +: : : : : : : : : +- CometProject +: : : : : : : : : +- CometFilter +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : : : +- CometBroadcastExchange +: : : : : : : : +- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : : : +- BroadcastExchange +: : : : : : +- CometColumnarToRow +: : : : : : +- CometHashAggregate +: : : : : : +- CometExchange +: : : : : : +- CometHashAggregate +: : : : : : +- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : :- CometProject +: : : : : : : : : +- CometFilter +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : : +- CometBroadcastExchange +: : : : : : : : +- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : : +- BroadcastExchange +: : : : : +- CometColumnarToRow +: : : : : +- CometHashAggregate +: : : : : +- CometExchange +: : : : : +- CometHashAggregate +: : : : : +- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : +- BroadcastExchange +: : : : +- CometColumnarToRow +: : : : +- CometHashAggregate +: : : : +- CometExchange +: : : : +- CometHashAggregate +: : : : +- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- BroadcastExchange +: : : +- CometColumnarToRow +: : : +- CometHashAggregate +: : : +- CometExchange +: : : +- CometHashAggregate +: : : +- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.store +: : +- BroadcastExchange +: : +- CometColumnarToRow +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometHashAggregate +: +- CometExchange +: +- CometHashAggregate +: +- CometProject +: +- CometBroadcastHashJoin +: :- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.time_dim +: +- CometBroadcastExchange +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.store ++- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 192 out of 206 eligible operators (93%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/simplified.txt index 5acfeba131..cef8e0e760 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/simplified.txt @@ -6,9 +6,9 @@ WholeStageCodegen (8) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h8_30_to_9,count,count(1)] + CometHashAggregate [count] [h8_30_to_9,count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -19,25 +19,25 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h9_to_9_30,count,count(1)] + CometHashAggregate [count] [h9_to_9_30,count(1)] CometExchange #6 CometHashAggregate [count] CometProject @@ -48,19 +48,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 WholeStageCodegen (2) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h9_30_to_10,count,count(1)] + CometHashAggregate [count] [h9_30_to_10,count(1)] CometExchange #9 CometHashAggregate [count] CometProject @@ -71,19 +71,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h10_to_10_30,count,count(1)] + CometHashAggregate [count] [h10_to_10_30,count(1)] CometExchange #12 CometHashAggregate [count] CometProject @@ -94,19 +94,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 WholeStageCodegen (4) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h10_30_to_11,count,count(1)] + CometHashAggregate [count] [h10_30_to_11,count(1)] CometExchange #15 CometHashAggregate [count] CometProject @@ -117,19 +117,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h11_to_11_30,count,count(1)] + CometHashAggregate [count] [h11_to_11_30,count(1)] CometExchange #18 CometHashAggregate [count] CometProject @@ -140,19 +140,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 WholeStageCodegen (6) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h11_30_to_12,count,count(1)] + CometHashAggregate [count] [h11_30_to_12,count(1)] CometExchange #21 CometHashAggregate [count] CometProject @@ -163,19 +163,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 WholeStageCodegen (7) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h12_to_12_30,count,count(1)] + CometHashAggregate [count] [h12_to_12_30,count(1)] CometExchange #24 CometHashAggregate [count] CometProject @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/explain.txt index 888fbc116c..22dd1d2dd8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/explain.txt @@ -1,147 +1,165 @@ == Physical Plan == -TakeOrderedAndProject (28) -+- * Project (27) - +- * Filter (26) - +- Window (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometExchange (22) - +- CometHashAggregate (21) - +- CometExchange (20) - +- CometHashAggregate (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Electronics ,Sports ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (computers ,stereo ,football )) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Men ,Jewelry ,Women ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(4) CometFilter -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Right output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_item_sk#1], [ss_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(11) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] - -(12) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(15) CometFilter -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) - -(16) CometBroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(17) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Right output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [ss_store_sk#6], [s_store_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14], [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] - -(19) CometHashAggregate -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] - -(20) CometExchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#15] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] - -(22) CometExchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(23) CometSort -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17], [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST] - -(24) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] - -(25) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] - -(26) Filter [codegen id : 2] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17, avg_monthly_sales#18] -Condition : CASE WHEN NOT (avg_monthly_sales#18 = 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END - -(27) Project [codegen id : 2] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17, avg_monthly_sales#18] - -(28) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#18) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] +(3) CometProject +Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) AS i_class#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) AS i_category#7] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Condition : (isnotnull(ss_item_sk#8) AND isnotnull(ss_store_sk#9)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] + +(7) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7] +Right output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight + +(8) CometProject +Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) + +(11) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] + +(12) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] + +(13) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight + +(14) CometProject +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) + +(17) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] + +(18) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight + +(19) CometProject +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] + +(20) CometHashAggregate +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] + +(21) CometExchange +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometHashAggregate +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] + +(23) CometExchange +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] + +(26) Window +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] + +(27) Filter [codegen id : 2] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN NOT (avg_monthly_sales#21 = 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END + +(28) Project [codegen id : 2] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] + +(29) TakeOrderedAndProject +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [(sum_sales#19 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/extended.txt new file mode 100644 index 0000000000..44fd48691b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/extended.txt @@ -0,0 +1,31 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/simplified.txt index 69ace610ec..56394060ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/simplified.txt @@ -5,28 +5,29 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category] + CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] + CometFilter [i_item_sk,i_brand,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/explain.txt index 41ec0fbbe3..da194f2825 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/explain.txt @@ -1,41 +1,47 @@ == Physical Plan == -* ColumnarToRow (4) -+- CometProject (3) +* Project (4) ++- * CometColumnarToRow (3) +- CometFilter (2) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (1) + +- CometNativeScan parquet spark_catalog.default.reason (1) -(1) CometNativeScan: `spark_catalog`.`default`.`reason` +(1) CometNativeScan parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] -Arguments: [r_reason_sk#1] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)] +ReadSchema: struct (2) CometFilter Input [1]: [r_reason_sk#1] Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) -(3) CometProject +(3) CometColumnarToRow [codegen id : 1] Input [1]: [r_reason_sk#1] -Arguments: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6], [CASE WHEN (Subquery scalar-subquery#7, [id=#8].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_net_paid) END AS bucket1#2, CASE WHEN (Subquery scalar-subquery#9, [id=#10].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_net_paid) END AS bucket2#3, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket3#4, CASE WHEN (Subquery scalar-subquery#13, [id=#14].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_net_paid) END AS bucket4#5, CASE WHEN (Subquery scalar-subquery#15, [id=#16].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_net_paid) END AS bucket5#6] -(4) ColumnarToRow [codegen id : 1] -Input [5]: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6] +(4) Project [codegen id : 1] +Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] +Input [1]: [r_reason_sk#1] ===== Subqueries ===== -Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#7, [id=#8] -* ColumnarToRow (12) +Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] +* CometColumnarToRow (12) +- CometProject (11) +- CometHashAggregate (10) +- CometExchange (9) +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) + +- CometNativeScan parquet spark_catalog.default.store_sales (5) -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)] +ReadSchema: struct (6) CometFilter Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] @@ -63,27 +69,30 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(Unscal Input [3]: [count(1)#26, avg(ss_ext_discount_amt)#27, avg(ss_net_paid)#28] Arguments: [mergedValue#29], [named_struct(count(1), count(1)#26, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#27, avg(ss_net_paid), avg(ss_net_paid)#28) AS mergedValue#29] -(12) ColumnarToRow [codegen id : 1] +(12) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#29] -Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] +Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] -Subquery:3 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] +Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] -Subquery:4 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (20) +Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] +* CometColumnarToRow (20) +- CometProject (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) + +- CometNativeScan parquet spark_catalog.default.store_sales (13) -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] -Arguments: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] +ReadSchema: struct (14) CometFilter Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] @@ -111,27 +120,30 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(Unscal Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] -(20) ColumnarToRow [codegen id : 1] +(20) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#42] -Subquery:5 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] -Subquery:6 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] -Subquery:7 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* ColumnarToRow (28) +Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] +* CometColumnarToRow (28) +- CometProject (27) +- CometHashAggregate (26) +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (21) + +- CometNativeScan parquet spark_catalog.default.store_sales (21) -(21) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(21) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] -Arguments: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] +ReadSchema: struct (22) CometFilter Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] @@ -159,27 +171,30 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(Unscal Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] -(28) ColumnarToRow [codegen id : 1] +(28) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#55] -Subquery:8 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] -Subquery:9 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] -Subquery:10 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#13, [id=#14] -* ColumnarToRow (36) +Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (36) +- CometProject (35) +- CometHashAggregate (34) +- CometExchange (33) +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (29) + +- CometNativeScan parquet spark_catalog.default.store_sales (29) -(29) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(29) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] -Arguments: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] +ReadSchema: struct (30) CometFilter Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] @@ -207,27 +222,30 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(Unscal Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] -(36) ColumnarToRow [codegen id : 1] +(36) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#68] -Subquery:11 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] +Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:12 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] +Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:13 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#15, [id=#16] -* ColumnarToRow (44) +Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] +* CometColumnarToRow (44) +- CometProject (43) +- CometHashAggregate (42) +- CometExchange (41) +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (37) + +- CometNativeScan parquet spark_catalog.default.store_sales (37) -(37) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(37) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] -Arguments: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] +ReadSchema: struct (38) CometFilter Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] @@ -255,11 +273,11 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(Unscal Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#81] -Subquery:14 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] +Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] -Subquery:15 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] +Subquery:15 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/extended.txt new file mode 100644 index 0000000000..53ba3252cf --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/extended.txt @@ -0,0 +1,61 @@ + Project [COMET: ] +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: +- ReusedSubquery ++- CometColumnarToRow + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/simplified.txt index 750b92722f..9593d6ebd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/simplified.txt @@ -1,71 +1,71 @@ WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [bucket1,bucket2,bucket3,bucket4,bucket5] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #1 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #1 - ReusedSubquery [mergedValue] #1 - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #2 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #2 - ReusedSubquery [mergedValue] #2 - Subquery #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #3 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #3 - ReusedSubquery [mergedValue] #3 - Subquery #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #4 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #4 - ReusedSubquery [mergedValue] #4 - Subquery #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #5 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #5 - ReusedSubquery [mergedValue] #5 + Project + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #1 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #1 + ReusedSubquery [mergedValue] #1 + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #2 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #2 + ReusedSubquery [mergedValue] #2 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #3 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #3 + ReusedSubquery [mergedValue] #3 + Subquery #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #4 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #4 + ReusedSubquery [mergedValue] #4 + Subquery #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #5 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #5 + ReusedSubquery [mergedValue] #5 + CometColumnarToRow + InputAdapter CometFilter [r_reason_sk] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/explain.txt index 6390d2d5eb..73b54f439b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * Project (47) +- * BroadcastNestedLoopJoin Inner BuildRight (46) - :- * ColumnarToRow (25) + :- * CometColumnarToRow (25) : +- CometHashAggregate (24) : +- CometExchange (23) : +- CometHashAggregate (22) @@ -13,21 +13,21 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (16) + : +- CometNativeScan parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) - +- * ColumnarToRow (44) + +- * CometColumnarToRow (44) +- CometHashAggregate (43) +- CometExchange (42) +- CometHashAggregate (41) @@ -39,18 +39,21 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) + : +- CometNativeScan parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] @@ -60,9 +63,12 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)] +ReadSchema: struct (5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] @@ -85,9 +91,12 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] -Arguments: [t_time_sk#7, t_hour#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)] +ReadSchema: struct (11) CometFilter Input [2]: [t_time_sk#7, t_hour#8] @@ -110,9 +119,12 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometNativeScan: `spark_catalog`.`default`.`web_page` +(16) CometNativeScan parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] -Arguments: [wp_web_page_sk#9, wp_char_count#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [wp_web_page_sk#9, wp_char_count#10] @@ -148,12 +160,15 @@ Input [1]: [count#11] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 2] +(25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(26) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] +ReadSchema: struct (27) CometFilter Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] @@ -175,9 +190,12 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] -Arguments: [t_time_sk#18, t_hour#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)] +ReadSchema: struct (33) CometFilter Input [2]: [t_time_sk#18, t_hour#19] @@ -225,7 +243,7 @@ Input [1]: [count#21] Keys: [] Functions [1]: [count(1)] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [pmc#22] (45) BroadcastExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/extended.txt new file mode 100644 index 0000000000..aa4016076b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/extended.txt @@ -0,0 +1,55 @@ +Project ++- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.time_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 48 out of 51 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/simplified.txt index a746bdf6f4..a13072cd91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (2) Project [amc,pmc] BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [amc,count,count(1)] + CometHashAggregate [count] [amc,count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -14,25 +14,25 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk,wp_char_count] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [pmc,count,count(1)] + CometHashAggregate [count] [pmc,count(1)] CometExchange #6 CometHashAggregate [count] CometProject @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/explain.txt index 0277b481b8..42aba65d76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/explain.txt @@ -1,215 +1,247 @@ == Physical Plan == -* ColumnarToRow (41) -+- CometSort (40) - +- CometColumnarExchange (39) - +- CometHashAggregate (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (13) - : : : : +- CometBroadcastHashJoin (12) - : : : : :- CometProject (7) - : : : : : +- CometBroadcastHashJoin (6) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (1) - : : : : : +- CometBroadcastExchange (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (3) - : : : : +- CometBroadcastExchange (11) - : : : : +- CometProject (10) - : : : : +- CometFilter (9) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : +- CometBroadcastExchange (16) - : : : +- CometFilter (15) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (14) - : : +- CometBroadcastExchange (22) - : : +- CometProject (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (19) - : +- CometBroadcastExchange (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (25) - +- CometBroadcastExchange (33) - +- CometProject (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`call_center` +* CometColumnarToRow (43) ++- CometSort (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometProject (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometFilter (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (4) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometNativeScan parquet spark_catalog.default.customer (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.household_demographics (32) + + +(1) CometNativeScan parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Arguments: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Condition : isnotnull(cc_call_center_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(4) CometFilter -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Right output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_sk#1], [cr_call_center_sk#6], Inner, BuildRight - -(7) CometProject -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(12) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [cr_returned_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(15) CometFilter -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) - -(16) CometBroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(17) CometBroadcastHashJoin -Left output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Right output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cr_returning_customer_sk#5], [c_customer_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16, ca_gmt_offset#17] - -(20) CometFilter -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) - -(21) CometProject -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(22) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(23) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Right output [1]: [ca_address_sk#16] -Arguments: [c_current_addr_sk#15], [ca_address_sk#16], Inner, BuildRight - -(24) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] - -(25) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(26) CometFilter -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) - -(27) CometBroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(28) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [c_current_cdemo_sk#13], [cd_demo_sk#18], Inner, BuildRight - -(29) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] - -(30) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21, hd_buy_potential#22] - -(31) CometFilter -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) - -(32) CometProject -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21], [hd_demo_sk#21] +(3) CometProject +Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cc_call_center_id#2, 16, true, false, true) AS cc_call_center_id#5, cc_name#3, cc_manager#4] + +(4) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] +PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_call_center_sk#7) AND isnotnull(cr_returning_customer_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [4]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4] +Right output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight + +(8) CometProject +Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) + +(11) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] + +(13) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(14) CometProject +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] + +(15) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) + +(17) CometBroadcastExchange +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] + +(18) CometBroadcastHashJoin +Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight + +(19) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] + +(20) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) + +(22) CometProject +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] + +(23) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] + +(24) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight + +(25) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] + +(26) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(27) CometFilter +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) + +(28) CometProject +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] + +(29) CometBroadcastExchange +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] + +(30) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight + +(31) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] + +(32) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#25, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#24)) + +(34) CometProject +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] + +(35) CometBroadcastExchange +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] + +(36) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight + +(37) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] -(33) CometBroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: [hd_demo_sk#21] +(38) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] -(34) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Right output [1]: [hd_demo_sk#21] -Arguments: [c_current_hdemo_sk#14], [hd_demo_sk#21], Inner, BuildRight - -(35) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] - -(36) CometHashAggregate -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] - -(37) CometExchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#23] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(39) CometExchange +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(38) CometHashAggregate -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#23] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] +(40) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] -(39) CometColumnarExchange -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] -Arguments: rangepartitioning(Returns_Loss#27 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(41) CometExchange +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: rangepartitioning(Returns_Loss#30 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(40) CometSort -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] -Arguments: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27], [Returns_Loss#27 DESC NULLS LAST] +(42) CometSort +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30], [Returns_Loss#30 DESC NULLS LAST] -(41) ColumnarToRow [codegen id : 1] -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] +(43) CometColumnarToRow [codegen id : 1] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/extended.txt new file mode 100644 index 0000000000..f3d3dc8b3b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.household_demographics + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/simplified.txt index 1cf9256a1a..36fc2b1b5b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] - CometColumnarExchange [Returns_Loss] #1 - CometHashAggregate [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,sum(UnscaledValue(cr_net_loss))] + CometExchange [Returns_Loss] #1 + CometHashAggregate [cd_marital_status,cd_education_status,sum] [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,sum(UnscaledValue(cr_net_loss))] CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - CometHashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,cr_net_loss] + CometHashAggregate [cr_net_loss] [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] @@ -18,26 +18,28 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/explain.txt index 8acc3929c9..56c8255ef7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (30) +* CometColumnarToRow (30) +- CometHashAggregate (29) +- CometExchange (28) +- CometHashAggregate (27) @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) + : : +- CometNativeScan parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,25 +23,32 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (9) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] @@ -64,17 +71,24 @@ Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -(9) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(9) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Arguments: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#8)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Condition : isnotnull(ws_item_sk#6) -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] +ReadSchema: struct (12) CometFilter Input [2]: [d_date_sk#9, d_date#10] @@ -154,6 +168,6 @@ Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -(30) ColumnarToRow [codegen id : 1] +(30) CometColumnarToRow [codegen id : 1] Input [1]: [Excess Discount Amount #16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/extended.txt new file mode 100644 index 0000000000..f9fb54b694 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/simplified.txt index 08b10d6a88..217c245298 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [Excess Discount Amount ,sum,sum(UnscaledValue(ws_ext_discount_amt))] + CometHashAggregate [sum] [Excess Discount Amount ,sum(UnscaledValue(ws_ext_discount_amt))] CometExchange #1 - CometHashAggregate [sum,ws_ext_discount_amt] + CometHashAggregate [ws_ext_discount_amt] [sum] CometProject [ws_ext_discount_amt] CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometProject [ws_ext_discount_amt,ws_sold_date_sk] @@ -11,22 +11,22 @@ WholeStageCodegen (1) CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] - CometHashAggregate [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,sum,count,avg(UnscaledValue(ws_ext_discount_amt))] + CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] CometExchange [ws_item_sk] #4 - CometHashAggregate [ws_item_sk,sum,count,ws_ext_discount_amt] + CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/explain.txt index ad03c691af..3f38adcd39 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (22) +* CometColumnarToRow (22) +- CometTakeOrderedAndProject (21) +- CometHashAggregate (20) +- CometExchange (19) @@ -11,21 +11,23 @@ : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) + : +- CometNativeScan parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (12) + +- CometNativeScan parquet spark_catalog.default.reason (12) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +ReadSchema: struct (2) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] @@ -39,9 +41,12 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)] +ReadSchema: struct (6) CometFilter Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] @@ -68,13 +73,16 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometNativeScan: `spark_catalog`.`default`.`reason` +(12) CometNativeScan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] -Arguments: [r_reason_sk#12, r_reason_desc#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_sk)] +ReadSchema: struct (13) CometFilter Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#13, 100, true, false, true) = reason 28 ) AND isnotnull(r_reason_sk#12)) (14) CometProject Input [2]: [r_reason_sk#12, r_reason_desc#13] @@ -109,8 +117,8 @@ Functions [1]: [sum(act_sales#14)] (21) CometTakeOrderedAndProject Input [2]: [ss_customer_sk#2, sumsales#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, 0, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] -(22) ColumnarToRow [codegen id : 1] +(22) CometColumnarToRow [codegen id : 1] Input [2]: [ss_customer_sk#2, sumsales#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/extended.txt new file mode 100644 index 0000000000..7203a678b0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/extended.txt @@ -0,0 +1,24 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 21 out of 21 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/simplified.txt index 4ede7ca586..c0b74010df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [ss_customer_sk,sumsales] - CometHashAggregate [ss_customer_sk,sumsales,sum,isEmpty,sum(act_sales)] + CometHashAggregate [sum,isEmpty] [ss_customer_sk,sumsales,sum(act_sales)] CometExchange [ss_customer_sk] #1 - CometHashAggregate [ss_customer_sk,sum,isEmpty,act_sales] + CometHashAggregate [act_sales] [ss_customer_sk,sum,isEmpty] CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] @@ -12,13 +12,13 @@ WholeStageCodegen (1) CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometExchange [ss_item_sk,ss_ticket_number] #2 CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometExchange [sr_item_sk,sr_ticket_number] #3 CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [r_reason_sk] #4 CometProject [r_reason_sk] CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/explain.txt index 1eb8827312..1caeb8ca76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/explain.txt @@ -1,49 +1,53 @@ == Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometNativeScan parquet spark_catalog.default.web_site (29) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] @@ -61,9 +65,11 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] -Arguments: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +ReadSchema: struct (7) CometProject Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] @@ -86,9 +92,11 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(12) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] -Arguments: [wr_order_number#12, wr_returned_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +ReadSchema: struct (13) CometProject Input [2]: [wr_order_number#12, wr_returned_date_sk#13] @@ -107,9 +115,12 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct (18) CometFilter Input [2]: [d_date_sk#14, d_date#15] @@ -132,13 +143,16 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#16)) (25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] @@ -157,13 +171,16 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometNativeScan: `spark_catalog`.`default`.`web_site` +(29) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] -Arguments: [web_site_sk#18, web_company_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct (30) CometFilter Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#19, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#18)) (31) CometProject Input [2]: [web_site_sk#18, web_company_name#19] @@ -187,7 +204,7 @@ Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Keys [1]: [ws_order_number#5] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] -(36) ColumnarToRow [codegen id : 1] +(36) CometColumnarToRow [codegen id : 1] Input [3]: [ws_order_number#5, sum#20, sum#21] (37) HashAggregate [codegen id : 1] @@ -204,14 +221,15 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] Results [3]: [sum#20, sum#21, count#25] -(39) Exchange +(39) CometColumnarExchange Input [3]: [sum#20, sum#21, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 2] +(40) CometHashAggregate Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#23,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/extended.txt new file mode 100644 index 0000000000..e5c7b8688a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/simplified.txt index bafb48b0bd..feab73bcd6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/simplified.txt @@ -1,44 +1,45 @@ WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [ws_order_number,sum,sum,ws_ext_ship_cost,ws_net_profit] - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/explain.txt index 2c87988976..39f45f7a74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/explain.txt @@ -1,62 +1,66 @@ == Physical Plan == -* HashAggregate (53) -+- Exchange (52) - +- * HashAggregate (51) - +- * HashAggregate (50) - +- * ColumnarToRow (49) - +- CometHashAggregate (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometSortMergeJoin (29) - : : : :- CometSortMergeJoin (15) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometProject (14) - : : : : +- CometSortMergeJoin (13) - : : : : :- CometSort (10) - : : : : : +- CometExchange (9) - : : : : : +- CometProject (8) - : : : : : +- CometFilter (7) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : : +- CometSort (12) - : : : : +- ReusedExchange (11) - : : : +- CometProject (28) - : : : +- CometSortMergeJoin (27) - : : : :- CometSort (20) - : : : : +- CometExchange (19) - : : : : +- CometProject (18) - : : : : +- CometFilter (17) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (16) - : : : +- CometProject (26) - : : : +- CometSortMergeJoin (25) - : : : :- CometSort (22) - : : : : +- ReusedExchange (21) - : : : +- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometBroadcastExchange (33) - : : +- CometProject (32) - : : +- CometFilter (31) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - : +- CometBroadcastExchange (39) - : +- CometProject (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - +- CometBroadcastExchange (45) - +- CometProject (44) - +- CometFilter (43) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* CometColumnarToRow (54) ++- CometHashAggregate (53) + +- CometColumnarExchange (52) + +- * HashAggregate (51) + +- * HashAggregate (50) + +- * CometColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometBroadcastHashJoin (46) + :- CometProject (41) + : +- CometBroadcastHashJoin (40) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometSortMergeJoin (29) + : : : :- CometSortMergeJoin (15) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometProject (14) + : : : : +- CometSortMergeJoin (13) + : : : : :- CometSort (10) + : : : : : +- CometExchange (9) + : : : : : +- CometProject (8) + : : : : : +- CometFilter (7) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) + : : : : +- CometSort (12) + : : : : +- ReusedExchange (11) + : : : +- CometProject (28) + : : : +- CometSortMergeJoin (27) + : : : :- CometSort (20) + : : : : +- CometExchange (19) + : : : : +- CometProject (18) + : : : : +- CometFilter (17) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (16) + : : : +- CometProject (26) + : : : +- CometSortMergeJoin (25) + : : : :- CometSort (22) + : : : : +- ReusedExchange (21) + : : : +- CometSort (24) + : : : +- ReusedExchange (23) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : +- CometBroadcastExchange (39) + : +- CometProject (38) + : +- CometFilter (37) + : +- CometNativeScan parquet spark_catalog.default.customer_address (36) + +- CometBroadcastExchange (45) + +- CometProject (44) + +- CometFilter (43) + +- CometNativeScan parquet spark_catalog.default.web_site (42) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] @@ -74,9 +78,12 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Arguments: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] +ReadSchema: struct (7) CometFilter Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] @@ -115,9 +122,12 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(16) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] -Arguments: [wr_order_number#13, wr_returned_date_sk#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number)] +ReadSchema: struct (17) CometFilter Input [2]: [wr_order_number#13, wr_returned_date_sk#14] @@ -172,9 +182,12 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#20] -Arguments: [d_date_sk#19, d_date#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct (31) CometFilter Input [2]: [d_date_sk#19, d_date#20] @@ -197,13 +210,16 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(36) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#21, ca_state#22] -Arguments: [ca_address_sk#21, ca_state#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (37) CometFilter Input [2]: [ca_address_sk#21, ca_state#22] -Condition : ((isnotnull(ca_state#22) AND (ca_state#22 = IL)) AND isnotnull(ca_address_sk#21)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#21)) (38) CometProject Input [2]: [ca_address_sk#21, ca_state#22] @@ -222,13 +238,16 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometNativeScan: `spark_catalog`.`default`.`web_site` +(42) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#23, web_company_name#24] -Arguments: [web_site_sk#23, web_company_name#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct (43) CometFilter Input [2]: [web_site_sk#23, web_company_name#24] -Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri )) AND isnotnull(web_site_sk#23)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#24, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#23)) (44) CometProject Input [2]: [web_site_sk#23, web_company_name#24] @@ -252,7 +271,7 @@ Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Keys [1]: [ws_order_number#4] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -(49) ColumnarToRow [codegen id : 1] +(49) CometColumnarToRow [codegen id : 1] Input [3]: [ws_order_number#4, sum#25, sum#26] (50) HashAggregate [codegen id : 1] @@ -269,14 +288,15 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] Results [3]: [sum#25, sum#26, count#30] -(52) Exchange +(52) CometColumnarExchange Input [3]: [sum#25, sum#26, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(53) HashAggregate [codegen id : 2] +(53) CometHashAggregate Input [3]: [sum#25, sum#26, count#30] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] -Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#27,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#28,17,2) AS total net profit #33] + +(54) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #31, total shipping cost #32, total net profit #33] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/extended.txt new file mode 100644 index 0000000000..cbba03d28d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/extended.txt @@ -0,0 +1,65 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 59 out of 61 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/simplified.txt index 4b35a28ce4..15f29c507c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/simplified.txt @@ -1,57 +1,58 @@ WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [ws_order_number,sum,sum,ws_ext_ship_cost,ws_net_profit] - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometProject [ws_order_number] - CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometProject [wr_order_number] - CometSortMergeJoin [wr_order_number,ws_order_number] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] + CometProject [wr_order_number] + CometSortMergeJoin [wr_order_number,ws_order_number] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/explain.txt index 16424de5f3..db701aae34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (25) +* CometColumnarToRow (25) +- CometHashAggregate (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -11,24 +11,27 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) + : +- CometNativeScan parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometNativeScan: `spark_catalog`.`default`.`store` (16) + +- CometNativeScan parquet spark_catalog.default.store (16) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] @@ -38,9 +41,12 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)] +ReadSchema: struct (5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] @@ -63,9 +69,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Arguments: [t_time_sk#7, t_hour#8, t_minute#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] @@ -88,9 +97,12 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometNativeScan: `spark_catalog`.`default`.`store` +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] -Arguments: [s_store_sk#10, s_store_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [s_store_sk#10, s_store_name#11] @@ -126,6 +138,6 @@ Input [1]: [count#12] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [1]: [count(1)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/extended.txt new file mode 100644 index 0000000000..170d3b7879 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/simplified.txt index 1996d860c8..b40f41659f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/simplified.txt @@ -1,7 +1,7 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [count(1),count,count(1)] + CometHashAggregate [count] [count(1),count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/explain.txt index 2c42ae9d7f..b5a88b6563 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (25) +* CometColumnarToRow (25) +- CometHashAggregate (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -11,28 +11,34 @@ : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : :- CometNativeScan parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (2) + : +- CometNativeScan parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) - :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (12) + :- CometNativeScan parquet spark_catalog.default.catalog_sales (12) +- ReusedExchange (13) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +ReadSchema: struct -(2) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(2) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (3) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -73,9 +79,12 @@ Functions: [] Input [2]: [customer_sk#6, item_sk#7] Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(12) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] -Arguments: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +ReadSchema: struct (13) ReusedExchange [Reuses operator id: 5] Output [1]: [d_date_sk#11] @@ -130,6 +139,6 @@ Input [3]: [sum#14, sum#15, sum#16] Keys: [] Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/extended.txt new file mode 100644 index 0000000000..07252f3fb5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/extended.txt @@ -0,0 +1,30 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometNativeScan parquet spark_catalog.default.catalog_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 27 out of 27 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/simplified.txt index 3a2bf7df1d..18199546d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [store_only,catalog_only,store_and_catalog,sum,sum,sum,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum] [store_only,catalog_only,store_and_catalog,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] CometExchange #1 - CometHashAggregate [sum,sum,sum,customer_sk,customer_sk] + CometHashAggregate [customer_sk,customer_sk] [sum,sum,sum] CometProject [customer_sk,customer_sk] CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] CometSort [customer_sk,item_sk] @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometHashAggregate [ss_customer_sk,ss_item_sk] CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] CometExchange [cs_bill_customer_sk,cs_item_sk] #4 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/explain.txt index 769ec05b5f..4966606a2a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/explain.txt @@ -1,130 +1,145 @@ == Physical Plan == -* ColumnarToRow (25) -+- CometProject (24) - +- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (26) ++- CometProject (25) + +- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(22) CometColumnarExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +(23) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(23) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST] +(24) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(24) CometProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(25) CometProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] -(25) ColumnarToRow [codegen id : 3] -Input [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(26) CometColumnarToRow [codegen id : 3] +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/extended.txt new file mode 100644 index 0000000000..41355f4433 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometProject + +- CometSort + +- CometColumnarExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 22 out of 24 eligible operators (91%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/simplified.txt index f4bffe18d8..a848d89c1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] @@ -9,23 +9,24 @@ WholeStageCodegen (3) InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] CometExchange [i_class] #2 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/explain.txt index 8c3bfd0df7..3447b6efa3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/explain.txt @@ -1,41 +1,52 @@ == Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.call_center (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -54,73 +65,104 @@ Arguments: [cs_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -(8) ReusedExchange [Reuses operator id: 5] +(8) CometNativeScan parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct -(9) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight +(9) CometFilter +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) (10) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [cc_call_center_sk#10, cc_name#11] +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] (12) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: [cs_call_center_sk#2], [cc_call_center_sk#10], Inner, BuildRight +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight (13) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#11, cc_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) CometHashAggregate -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +Input [2]: [cc_call_center_sk#11, cc_name#12] +Condition : isnotnull(cc_call_center_sk#11) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cc_call_center_sk#11, cc_name#12] + +(17) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cs_call_center_sk#2], [cc_call_center_sk#11], Inner, BuildRight + +(18) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_call_center_sk#11, cc_name#12] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] + +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) + +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(23) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight + +(24) CometProject +Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12, d_date_sk#13] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] + +(25) CometHashAggregate +Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(21) CometExchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(23) CometTakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST,sm_type#9 ASC NULLS FIRST,cc_name#11 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#20,sm_type#9,cc_name#11,30 days #21,31 - 60 days #22,61 - 90 days #23,91 - 120 days #24,>120 days #25]), [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25], 100, [substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,cc_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,cc_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] -(24) ColumnarToRow [codegen id : 1] -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/extended.txt new file mode 100644 index 0000000000..c075163849 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.ship_mode + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/simplified.txt index ddd2b3d421..98a215dbc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] CometExchange [_groupingexpression,sm_type,cc_name] #1 - CometHashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum,cs_ship_date_sk,cs_sold_date_sk] + CometHashAggregate [cs_ship_date_sk,cs_sold_date_sk] [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] @@ -14,13 +14,18 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [cc_call_center_sk,cc_name] #2 - CometBroadcastExchange [d_date_sk] #3 + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [cc_call_center_sk,cc_name] #4 + CometFilter [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/explain.txt index a1a0e06f4f..c353301170 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/explain.txt @@ -1,60 +1,69 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (30) + : : +- * BroadcastHashJoin Inner BuildRight (29) : : :- * Filter (13) : : : +- * HashAggregate (12) - : : : +- Exchange (11) - : : : +- * HashAggregate (10) - : : : +- * ColumnarToRow (9) + : : : +- * CometColumnarToRow (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- BroadcastExchange (27) - : : +- * Filter (26) - : : +- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * HashAggregate (22) - : : +- Exchange (21) - : : +- * HashAggregate (20) - : : +- * ColumnarToRow (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometFilter (15) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (14) - : : +- ReusedExchange (16) - : +- BroadcastExchange (34) - : +- * ColumnarToRow (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (30) - +- BroadcastExchange (40) - +- * ColumnarToRow (39) - +- CometFilter (38) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (37) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (28) + : : +- * Filter (27) + : : +- * HashAggregate (26) + : : +- * CometColumnarToRow (25) + : : +- CometColumnarExchange (24) + : : +- * HashAggregate (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (14) + : : +- ReusedExchange (16) + : +- BroadcastExchange (35) + : +- * CometColumnarToRow (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometNativeScan parquet spark_catalog.default.store (31) + +- BroadcastExchange (42) + +- * CometColumnarToRow (41) + +- CometProject (40) + +- CometFilter (39) + +- CometNativeScan parquet spark_catalog.default.customer (38) + + +(1) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#4)] +PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_year#6] @@ -77,163 +86,176 @@ Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -(9) ColumnarToRow [codegen id : 1] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] - -(10) HashAggregate [codegen id : 1] +(9) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -(11) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(10) CometExchange +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) HashAggregate [codegen id : 7] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +(11) CometColumnarToRow [codegen id : 5] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] + +(12) HashAggregate [codegen id : 5] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#10, sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] +Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#8] +Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#9, sr_store_sk#2 AS ctr_store_sk#10, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#8,17,2) AS ctr_total_return#11] -(13) Filter [codegen id : 7] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) +(13) Filter [codegen id : 5] +Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11] +Condition : isnotnull(ctr_total_return#11) -(14) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Arguments: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] +(14) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#15)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct (15) CometFilter -Input [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_store_sk#14) +Input [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Condition : isnotnull(sr_store_sk#13) (16) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [sr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight +Left output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [sr_returned_date_sk#15], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [5]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16, d_date_sk#17] -Arguments: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15], [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] - -(19) ColumnarToRow [codegen id : 2] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] - -(20) HashAggregate [codegen id : 2] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] -Keys [2]: [sr_customer_sk#13, sr_store_sk#14] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#15))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] - -(21) Exchange -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] -Arguments: hashpartitioning(sr_customer_sk#13, sr_store_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(22) HashAggregate [codegen id : 3] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] -Keys [2]: [sr_customer_sk#13, sr_store_sk#14] -Functions [1]: [sum(UnscaledValue(sr_return_amt#15))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#15))#9] -Results [2]: [sr_store_sk#14 AS ctr_store_sk#20, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#9,17,2) AS ctr_total_return#21] - -(23) HashAggregate [codegen id : 3] -Input [2]: [ctr_store_sk#20, ctr_total_return#21] -Keys [1]: [ctr_store_sk#20] -Functions [1]: [partial_avg(ctr_total_return#21)] -Aggregate Attributes [2]: [sum#22, count#23] -Results [3]: [ctr_store_sk#20, sum#24, count#25] - -(24) Exchange -Input [3]: [ctr_store_sk#20, sum#24, count#25] -Arguments: hashpartitioning(ctr_store_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(25) HashAggregate [codegen id : 4] -Input [3]: [ctr_store_sk#20, sum#24, count#25] -Keys [1]: [ctr_store_sk#20] -Functions [1]: [avg(ctr_total_return#21)] -Aggregate Attributes [1]: [avg(ctr_total_return#21)#26] -Results [2]: [(avg(ctr_total_return#21)#26 * 1.2) AS (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] - -(26) Filter [codegen id : 4] -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#27) - -(27) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] +Input [5]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15, d_date_sk#16] +Arguments: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14], [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] + +(19) CometHashAggregate +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] +Keys [2]: [sr_customer_sk#12, sr_store_sk#13] +Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#14))] + +(20) CometExchange +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] +Arguments: hashpartitioning(sr_customer_sk#12, sr_store_sk#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] + +(22) HashAggregate [codegen id : 1] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] +Keys [2]: [sr_customer_sk#12, sr_store_sk#13] +Functions [1]: [sum(UnscaledValue(sr_return_amt#14))] +Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#14))#8] +Results [2]: [sr_store_sk#13 AS ctr_store_sk#18, MakeDecimal(sum(UnscaledValue(sr_return_amt#14))#8,17,2) AS ctr_total_return#19] + +(23) HashAggregate [codegen id : 1] +Input [2]: [ctr_store_sk#18, ctr_total_return#19] +Keys [1]: [ctr_store_sk#18] +Functions [1]: [partial_avg(ctr_total_return#19)] +Aggregate Attributes [2]: [sum#20, count#21] +Results [3]: [ctr_store_sk#18, sum#22, count#23] + +(24) CometColumnarExchange +Input [3]: [ctr_store_sk#18, sum#22, count#23] +Arguments: hashpartitioning(ctr_store_sk#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(25) CometColumnarToRow [codegen id : 2] +Input [3]: [ctr_store_sk#18, sum#22, count#23] + +(26) HashAggregate [codegen id : 2] +Input [3]: [ctr_store_sk#18, sum#22, count#23] +Keys [1]: [ctr_store_sk#18] +Functions [1]: [avg(ctr_total_return#19)] +Aggregate Attributes [1]: [avg(ctr_total_return#19)#24] +Results [2]: [(avg(ctr_total_return#19)#24 * 1.2) AS (avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] + +(27) Filter [codegen id : 2] +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) + +(28) BroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [ctr_store_sk#20] +(29) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_store_sk#10] +Right keys [1]: [ctr_store_sk#18] Join type: Inner -Join condition: (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27) +Join condition: (cast(ctr_total_return#11 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25) -(29) Project [codegen id : 7] -Output [2]: [ctr_customer_sk#10, ctr_store_sk#11] -Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] +(30) Project [codegen id : 5] +Output [2]: [ctr_customer_sk#9, ctr_store_sk#10] +Input [5]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11, (avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] -(30) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#28, s_state#29] -Arguments: [s_store_sk#28, s_state#29] +(31) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#26, s_state#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct -(31) CometFilter -Input [2]: [s_store_sk#28, s_state#29] -Condition : ((isnotnull(s_state#29) AND (s_state#29 = TN)) AND isnotnull(s_store_sk#28)) +(32) CometFilter +Input [2]: [s_store_sk#26, s_state#27] +Condition : ((isnotnull(s_state#27) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#27, 2)) = TN)) AND isnotnull(s_store_sk#26)) -(32) CometProject -Input [2]: [s_store_sk#28, s_state#29] -Arguments: [s_store_sk#28], [s_store_sk#28] +(33) CometProject +Input [2]: [s_store_sk#26, s_state#27] +Arguments: [s_store_sk#26], [s_store_sk#26] -(33) ColumnarToRow [codegen id : 5] -Input [1]: [s_store_sk#28] +(34) CometColumnarToRow [codegen id : 3] +Input [1]: [s_store_sk#26] -(34) BroadcastExchange -Input [1]: [s_store_sk#28] +(35) BroadcastExchange +Input [1]: [s_store_sk#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [s_store_sk#28] +(36) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_store_sk#10] +Right keys [1]: [s_store_sk#26] Join type: Inner Join condition: None -(36) Project [codegen id : 7] -Output [1]: [ctr_customer_sk#10] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, s_store_sk#28] +(37) Project [codegen id : 5] +Output [1]: [ctr_customer_sk#9] +Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, s_store_sk#26] + +(38) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#28, c_customer_id#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(37) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#30, c_customer_id#31] -Arguments: [c_customer_sk#30, c_customer_id#31] +(39) CometFilter +Input [2]: [c_customer_sk#28, c_customer_id#29] +Condition : isnotnull(c_customer_sk#28) -(38) CometFilter -Input [2]: [c_customer_sk#30, c_customer_id#31] -Condition : isnotnull(c_customer_sk#30) +(40) CometProject +Input [2]: [c_customer_sk#28, c_customer_id#29] +Arguments: [c_customer_sk#28, c_customer_id#30], [c_customer_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#29, 16)) AS c_customer_id#30] -(39) ColumnarToRow [codegen id : 6] -Input [2]: [c_customer_sk#30, c_customer_id#31] +(41) CometColumnarToRow [codegen id : 4] +Input [2]: [c_customer_sk#28, c_customer_id#30] -(40) BroadcastExchange -Input [2]: [c_customer_sk#30, c_customer_id#31] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(42) BroadcastExchange +Input [2]: [c_customer_sk#28, c_customer_id#30] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_customer_sk#10] -Right keys [1]: [c_customer_sk#30] +(43) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_customer_sk#9] +Right keys [1]: [c_customer_sk#28] Join type: Inner Join condition: None -(42) Project [codegen id : 7] -Output [1]: [c_customer_id#31] -Input [3]: [ctr_customer_sk#10, c_customer_sk#30, c_customer_id#31] +(44) Project [codegen id : 5] +Output [1]: [c_customer_id#30] +Input [3]: [ctr_customer_sk#9, c_customer_sk#28, c_customer_id#30] -(43) TakeOrderedAndProject -Input [1]: [c_customer_id#31] -Arguments: 100, [c_customer_id#31 ASC NULLS FIRST], [c_customer_id#31] +(45) TakeOrderedAndProject +Input [1]: [c_customer_id#30] +Arguments: 100, [c_customer_id#30 ASC NULLS FIRST], [c_customer_id#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/extended.txt new file mode 100644 index 0000000000..674008234e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/extended.txt @@ -0,0 +1,50 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Filter + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(sr_customer_sk#1 AS ctr_customer_sk#2, sr_store_sk#3 AS ctr_store_sk#4, MakeDecimal(sum(UnscaledValue(sr_return_amt#5))#6,17,2) AS ctr_total_return#7)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- Filter + : : +- HashAggregate + : : +- CometColumnarToRow + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(sr_store_sk#8 AS ctr_store_sk#9, MakeDecimal(sum(UnscaledValue(sr_return_amt#10))#6,17,2) AS ctr_total_return#11)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 27 out of 43 eligible operators (62%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/simplified.txt index 2e40cb58cc..dea79929ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [c_customer_id] - WholeStageCodegen (7) + WholeStageCodegen (5) Project [c_customer_id] BroadcastHashJoin [ctr_customer_sk,c_customer_sk] Project [ctr_customer_sk] @@ -8,53 +8,51 @@ TakeOrderedAndProject [c_customer_id] BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)] Filter [ctr_total_return] HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #1 - WholeStageCodegen (1) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometColumnarToRow + InputAdapter + CometExchange [sr_customer_sk,sr_store_sk] #1 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #3 - WholeStageCodegen (4) + WholeStageCodegen (2) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - InputAdapter - Exchange [ctr_store_sk] #4 - WholeStageCodegen (3) - HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #5 - WholeStageCodegen (2) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #2 + CometColumnarToRow + InputAdapter + CometColumnarExchange [ctr_store_sk] #4 + WholeStageCodegen (1) + HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] + HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [sr_customer_sk,sr_store_sk] #5 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #2 InputAdapter BroadcastExchange #6 - WholeStageCodegen (5) - ColumnarToRow + WholeStageCodegen (3) + CometColumnarToRow InputAdapter CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #7 - WholeStageCodegen (6) - ColumnarToRow + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_customer_id] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id] + CometProject [c_customer_id] [c_customer_sk,c_customer_id] + CometFilter [c_customer_sk,c_customer_id] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/explain.txt index 7c92771145..d12565753a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/explain.txt @@ -1,61 +1,77 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (28) - : +- * ColumnarToRow (27) - : +- CometProject (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (34) - +- * ColumnarToRow (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] @@ -87,12 +103,15 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#13] @@ -106,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -119,96 +138,132 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct -(21) BroadcastHashJoin [codegen id : 5] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None -(22) Filter [codegen id : 5] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(23) Project [codegen id : 5] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15, ca_county#16] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_county#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] +ReadSchema: struct -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_county#16] -Condition : (ca_county#16 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#15)) +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) -(26) CometProject -Input [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15], [ca_address_sk#15] +(31) CometProject +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(27) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#15] +(32) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#17] -(28) BroadcastExchange -Input [1]: [ca_address_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(33) BroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(29) BroadcastHashJoin [codegen id : 5] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(30) Project [codegen id : 5] +(35) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] -(31) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(32) CometFilter -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Condition : isnotnull(cd_demo_sk#17) +(37) CometFilter +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) -(33) ColumnarToRow [codegen id : 4] -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(38) CometProject +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#20, 1)) AS cd_gender#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) AS cd_marital_status#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) AS cd_education_status#30, cd_purchase_estimate#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#24, 10)) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(34) BroadcastExchange -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(39) CometColumnarToRow [codegen id : 4] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(35) BroadcastHashJoin [codegen id : 5] +(40) BroadcastExchange +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] +Right keys [1]: [cd_demo_sk#19] Join type: Inner Join condition: None -(36) Project [codegen id : 5] -Output [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(42) Project [codegen id : 5] +Output [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(37) HashAggregate [codegen id : 5] -Input [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(43) HashAggregate [codegen id : 5] +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#26] -Results [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] +Aggregate Attributes [1]: [count#32] +Results [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] + +(44) CometColumnarExchange +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(38) Exchange -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Arguments: hashpartitioning(cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(45) CometColumnarToRow [codegen id : 6] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] -(39) HashAggregate [codegen id : 6] -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +(46) HashAggregate [codegen id : 6] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#28] -Results [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, count(1)#28 AS cnt1#29, cd_purchase_estimate#21, count(1)#28 AS cnt2#30, cd_credit_rating#22, count(1)#28 AS cnt3#31, cd_dep_count#23, count(1)#28 AS cnt4#32, cd_dep_employed_count#24, count(1)#28 AS cnt5#33, cd_dep_college_count#25, count(1)#28 AS cnt6#34] +Aggregate Attributes [1]: [count(1)#34] +Results [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, count(1)#34 AS cnt1#35, cd_purchase_estimate#23, count(1)#34 AS cnt2#36, cd_credit_rating#31, count(1)#34 AS cnt3#37, cd_dep_count#25, count(1)#34 AS cnt4#38, cd_dep_employed_count#26, count(1)#34 AS cnt5#39, cd_dep_college_count#27, count(1)#34 AS cnt6#40] -(40) TakeOrderedAndProject -Input [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] -Arguments: 100, [cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_education_status#20 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#22 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] +(47) TakeOrderedAndProject +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] +Arguments: 100, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/simplified.txt index 228afd2bac..e5aeaa20c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/simplified.txt @@ -1,54 +1,63 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/explain.txt index 72f8bb38d1..61872c61ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/explain.txt @@ -1,362 +1,411 @@ == Physical Plan == -TakeOrderedAndProject (66) -+- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (50) - : +- * BroadcastHashJoin Inner BuildRight (49) - : :- * Project (34) - : : +- * BroadcastHashJoin Inner BuildRight (33) - : : :- * Filter (17) - : : : +- * HashAggregate (16) - : : : +- Exchange (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- BroadcastExchange (32) - : : +- * HashAggregate (31) - : : +- Exchange (30) - : : +- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (18) - : : : +- ReusedExchange (20) - : : +- CometBroadcastExchange (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - : +- BroadcastExchange (48) - : +- * Filter (47) - : +- * HashAggregate (46) - : +- Exchange (45) - : +- * HashAggregate (44) - : +- * ColumnarToRow (43) - : +- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometFilter (36) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (35) - : : +- ReusedExchange (37) - : +- ReusedExchange (40) - +- BroadcastExchange (63) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * ColumnarToRow (59) - +- CometProject (58) - +- CometBroadcastHashJoin (57) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometFilter (52) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (51) - : +- ReusedExchange (53) - +- ReusedExchange (56) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (72) ++- * Project (71) + +- * BroadcastHashJoin Inner BuildRight (70) + :- * Project (55) + : +- * BroadcastHashJoin Inner BuildRight (54) + : :- * Project (36) + : : +- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (53) + : +- * Filter (52) + : +- * HashAggregate (51) + : +- * CometColumnarToRow (50) + : +- CometExchange (49) + : +- CometHashAggregate (48) + : +- CometProject (47) + : +- CometBroadcastHashJoin (46) + : :- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometProject (39) + : : : +- CometFilter (38) + : : : +- CometNativeScan parquet spark_catalog.default.customer (37) + : : +- CometBroadcastExchange (42) + : : +- CometFilter (41) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (40) + : +- ReusedExchange (45) + +- BroadcastExchange (69) + +- * HashAggregate (68) + +- * CometColumnarToRow (67) + +- CometExchange (66) + +- CometHashAggregate (65) + +- CometProject (64) + +- CometBroadcastHashJoin (63) + :- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (58) + : : +- CometFilter (57) + : : +- CometNativeScan parquet spark_catalog.default.customer (56) + : +- ReusedExchange (59) + +- ReusedExchange (62) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(13) ColumnarToRow [codegen id : 1] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(14) HashAggregate [codegen id : 1] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(15) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(16) HashAggregate [codegen id : 8] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(17) Filter [codegen id : 8] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(18) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Arguments: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] - -(19) CometFilter -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) - -(20) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(21) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Right output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Arguments: [c_customer_sk#20], [ss_customer_sk#28], Inner, BuildRight - -(22) CometProject -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31], [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] - -(24) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] - -(26) CometBroadcastHashJoin -Left output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ss_sold_date_sk#31], [d_date_sk#32], Inner, BuildRight - -(27) CometProject -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] -Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33], [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] - -(28) ColumnarToRow [codegen id : 2] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] - -(29) HashAggregate [codegen id : 2] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(30) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(31) HashAggregate [codegen id : 3] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [3]: [c_customer_id#21 AS customer_id#36, c_preferred_cust_flag#24 AS customer_preferred_cust_flag#37, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#38] - -(32) BroadcastExchange -Input [3]: [customer_id#36, customer_preferred_cust_flag#37, year_total#38] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] + +(17) HashAggregate [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22] +Results [2]: [c_customer_id#9 AS customer_id#23, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22,18,2) AS year_total#24] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#23, year_total#24] +Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) + +(19) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Condition : (isnotnull(c_customer_sk#25) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)))) + +(21) CometProject +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Arguments: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38], [c_customer_sk#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)) AS c_customer_id#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#29, 1)) AS c_preferred_cust_flag#36, c_birth_country#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#31, 13)) AS c_login#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#32, 50)) AS c_email_address#38] + +(22) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(23) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Right output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_sk#25], [ss_customer_sk#39], Inner, BuildRight + +(24) CometProject +Input [12]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(25) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] + +(28) CometBroadcastHashJoin +Left output [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#43], Inner, BuildRight + +(29) CometProject +Input [12]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] + +(30) CometHashAggregate +Input [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] + +(31) CometExchange +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] + +(33) HashAggregate [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22] +Results [3]: [c_customer_id#33 AS customer_id#46, c_preferred_cust_flag#36 AS customer_preferred_cust_flag#47, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22,18,2) AS year_total#48] + +(34) BroadcastExchange +Input [3]: [customer_id#46, customer_preferred_cust_flag#47, year_total#48] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#46] Join type: Inner Join condition: None -(34) Project [codegen id : 8] -Output [4]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38] -Input [5]: [customer_id#18, year_total#19, customer_id#36, customer_preferred_cust_flag#37, year_total#38] - -(35) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Arguments: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] +(36) Project [codegen id : 4] +Output [4]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48] +Input [5]: [customer_id#23, year_total#24, customer_id#46, customer_preferred_cust_flag#47, year_total#48] -(36) CometFilter -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_customer_id#40)) +(37) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(37) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(38) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Right output [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Arguments: [c_customer_sk#39], [ws_bill_customer_sk#47], Inner, BuildRight +(38) CometFilter +Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Condition : (isnotnull(c_customer_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#50, 16)))) (39) CometProject -Input [12]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50], [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(40) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#51, d_year#52] - -(41) CometBroadcastHashJoin -Left output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Right output [2]: [d_date_sk#51, d_year#52] -Arguments: [ws_sold_date_sk#50], [d_date_sk#51], Inner, BuildRight - -(42) CometProject -Input [12]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52], [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] - -(43) ColumnarToRow [codegen id : 4] -Input [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] - -(44) HashAggregate [codegen id : 4] -Input [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum#53] -Results [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] - -(45) Exchange -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(46) HashAggregate [codegen id : 5] -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55] -Results [2]: [c_customer_id#40 AS customer_id#56, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55,18,2) AS year_total#57] - -(47) Filter [codegen id : 5] -Input [2]: [customer_id#56, year_total#57] -Condition : (isnotnull(year_total#57) AND (year_total#57 > 0.00)) - -(48) BroadcastExchange -Input [2]: [customer_id#56, year_total#57] +Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Arguments: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62], [c_customer_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#50, 16)) AS c_customer_id#57, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#51, 20)) AS c_first_name#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#52, 30)) AS c_last_name#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#53, 1)) AS c_preferred_cust_flag#60, c_birth_country#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#55, 13)) AS c_login#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#56, 50)) AS c_email_address#62] + +(40) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#66)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(41) CometFilter +Input [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Condition : isnotnull(ws_bill_customer_sk#63) + +(42) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] + +(43) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62] +Right output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [c_customer_sk#49], [ws_bill_customer_sk#63], Inner, BuildRight + +(44) CometProject +Input [12]: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66], [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#67, d_year#68] + +(46) CometBroadcastHashJoin +Left output [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Right output [2]: [d_date_sk#67, d_year#68] +Arguments: [ws_sold_date_sk#66], [d_date_sk#67], Inner, BuildRight + +(47) CometProject +Input [12]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66, d_date_sk#67, d_year#68] +Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68], [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] + +(48) CometHashAggregate +Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] +Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] + +(49) CometExchange +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] +Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(50) CometColumnarToRow [codegen id : 2] +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] + +(51) HashAggregate [codegen id : 2] +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] +Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))#70] +Results [2]: [c_customer_id#57 AS customer_id#71, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))#70,18,2) AS year_total#72] + +(52) Filter [codegen id : 2] +Input [2]: [customer_id#71, year_total#72] +Condition : (isnotnull(year_total#72) AND (year_total#72 > 0.00)) + +(53) BroadcastExchange +Input [2]: [customer_id#71, year_total#72] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(49) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#56] +(54) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#71] Join type: Inner Join condition: None -(50) Project [codegen id : 8] -Output [5]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57] -Input [6]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, customer_id#56, year_total#57] - -(51) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Arguments: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] - -(52) CometFilter -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Condition : (isnotnull(c_customer_sk#58) AND isnotnull(c_customer_id#59)) - -(53) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(54) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Right output [4]: [ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Arguments: [c_customer_sk#58], [ws_bill_customer_sk#66], Inner, BuildRight - -(55) CometProject -Input [12]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] +(55) Project [codegen id : 4] +Output [5]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, year_total#72] +Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, customer_id#71, year_total#72] -(56) ReusedExchange [Reuses operator id: 25] -Output [2]: [d_date_sk#70, d_year#71] +(56) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(57) CometBroadcastHashJoin -Left output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Right output [2]: [d_date_sk#70, d_year#71] -Arguments: [ws_sold_date_sk#69], [d_date_sk#70], Inner, BuildRight +(57) CometFilter +Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Condition : (isnotnull(c_customer_sk#73) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#74, 16)))) (58) CometProject -Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69, d_date_sk#70, d_year#71] -Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] - -(59) ColumnarToRow [codegen id : 6] -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] - -(60) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum#72] -Results [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] - -(61) Exchange -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(62) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55] -Results [2]: [c_customer_id#59 AS customer_id#74, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55,18,2) AS year_total#75] - -(63) BroadcastExchange -Input [2]: [customer_id#74, year_total#75] +Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Arguments: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86], [c_customer_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#74, 16)) AS c_customer_id#81, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#75, 20)) AS c_first_name#82, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#76, 30)) AS c_last_name#83, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#77, 1)) AS c_preferred_cust_flag#84, c_birth_country#78, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#79, 13)) AS c_login#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#80, 50)) AS c_email_address#86] + +(59) ReusedExchange [Reuses operator id: 42] +Output [4]: [ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] + +(60) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86] +Right output [4]: [ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Arguments: [c_customer_sk#73], [ws_bill_customer_sk#87], Inner, BuildRight + +(61) CometProject +Input [12]: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Arguments: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90], [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] + +(62) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#91, d_year#92] + +(63) CometBroadcastHashJoin +Left output [10]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Right output [2]: [d_date_sk#91, d_year#92] +Arguments: [ws_sold_date_sk#90], [d_date_sk#91], Inner, BuildRight + +(64) CometProject +Input [12]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90, d_date_sk#91, d_year#92] +Arguments: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92], [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92] + +(65) CometHashAggregate +Input [10]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92] +Keys [8]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))] + +(66) CometExchange +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] +Arguments: hashpartitioning(c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(67) CometColumnarToRow [codegen id : 3] +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] + +(68) HashAggregate [codegen id : 3] +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] +Keys [8]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))#70] +Results [2]: [c_customer_id#81 AS customer_id#94, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))#70,18,2) AS year_total#95] + +(69) BroadcastExchange +Input [2]: [customer_id#94, year_total#95] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(64) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#74] +(70) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#94] Join type: Inner -Join condition: (CASE WHEN (year_total#57 > 0.00) THEN (year_total#75 / year_total#57) END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#38 / year_total#19) END) +Join condition: (CASE WHEN (year_total#72 > 0.00) THEN (year_total#95 / year_total#72) END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#48 / year_total#24) END) -(65) Project [codegen id : 8] -Output [1]: [customer_preferred_cust_flag#37] -Input [7]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57, customer_id#74, year_total#75] +(71) Project [codegen id : 4] +Output [1]: [customer_preferred_cust_flag#47] +Input [7]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, year_total#72, customer_id#94, year_total#95] -(66) TakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#37] -Arguments: 100, [customer_preferred_cust_flag#37 ASC NULLS FIRST], [customer_preferred_cust_flag#37] +(72) TakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#47] +Arguments: 100, [customer_preferred_cust_flag#47 ASC NULLS FIRST], [customer_preferred_cust_flag#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/extended.txt new file mode 100644 index 0000000000..2370e02423 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/extended.txt @@ -0,0 +1,82 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Filter + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#1 AS customer_id#2, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#3 - ss_ext_discount_amt#4)))#5,18,2) AS year_total#6)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#7 AS customer_id#8, c_preferred_cust_flag#9 AS customer_preferred_cust_flag#10, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#12)))#5,18,2) AS year_total#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#14 AS customer_id#15, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#16 - ws_ext_discount_amt#17)))#18,18,2) AS year_total#19)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#20 AS customer_id#21, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#22 - ws_ext_discount_amt#23)))#18,18,2) AS year_total#24)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 60 out of 76 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/simplified.txt index 64b628845f..d19e0db21c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [customer_preferred_cust_flag] - WholeStageCodegen (8) + WholeStageCodegen (4) Project [customer_preferred_cust_flag] BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] @@ -8,78 +8,76 @@ TakeOrderedAndProject [customer_preferred_cust_flag] BroadcastHashJoin [customer_id,customer_id] Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (1) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #7 - WholeStageCodegen (5) + WholeStageCodegen (2) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (4) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #3 + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) + BroadcastExchange #10 + WholeStageCodegen (3) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #10 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #6 + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/explain.txt index 1f55fd0951..66020a3b06 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/explain.txt @@ -1,120 +1,137 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * Sort (19) - +- Exchange (18) - +- * HashAggregate (17) - +- Exchange (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) ColumnarToRow [codegen id : 1] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(15) HashAggregate [codegen id : 1] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(16) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) HashAggregate [codegen id : 2] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] + +(19) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(18) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(20) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) Sort [codegen id : 3] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(22) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] -(21) Project [codegen id : 4] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(23) Project [codegen id : 3] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(24) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/extended.txt new file mode 100644 index 0000000000..d0aa7cb0b1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/extended.txt @@ -0,0 +1,26 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_desc#1, i_category#2, i_class#3, i_current_price#4, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#5))#6,17,2) AS itemrevenue#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#5))#6,17,2) AS _w0#8, i_item_id#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 18 out of 22 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/simplified.txt index dc9d0df5f1..c69115af7f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/simplified.txt @@ -1,30 +1,30 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] InputAdapter Window [_w0,i_class] - WholeStageCodegen (3) - Sort [i_class] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/explain.txt index 07b08a045c..7e70ad90e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/explain.txt @@ -1,12 +1,12 @@ == Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) +* HashAggregate (34) ++- * CometColumnarToRow (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (19) : : +- CometBroadcastHashJoin (18) : : :- CometProject (13) @@ -14,37 +14,45 @@ : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) + : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) + +- CometBroadcastExchange (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.household_demographics (26) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] +ReadSchema: struct (2) CometFilter Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometNativeScan: `spark_catalog`.`default`.`store` +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [s_store_sk#11] @@ -63,116 +71,130 @@ Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (TX,OH) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (OR,NM,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (VA,TX,MS))) (10) CometProject Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) AS ca_state#15] (11) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] (12) CometBroadcastHashJoin Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight (13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#16, d_year#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#10], [d_date_sk#15], Inner, BuildRight +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(20) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) - -(22) CometBroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#17], Inner, ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight - -(24) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] - -(25) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = 2 yr Degree )))) -(26) CometFilter -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) +(22) CometProject +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#22] -(27) CometBroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(28) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Right output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#20], Inner, (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))), BuildRight - -(29) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] +(24) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight + +(25) CometProject +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] + +(26) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#23, hd_dep_count#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] +ReadSchema: struct + +(27) CometFilter +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) + +(28) CometBroadcastExchange +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] + +(29) CometBroadcastHashJoin +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight + +(30) CometProject +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(30) ColumnarToRow [codegen id : 1] -Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] - -(31) HashAggregate [codegen id : 1] +(31) CometHashAggregate Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Results [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -(32) Exchange -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] +(32) CometExchange +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(33) CometColumnarToRow [codegen id : 1] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] -(33) HashAggregate [codegen id : 2] -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(34) HashAggregate [codegen id : 1] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [4]: [avg(ss_quantity#5)#36, avg(UnscaledValue(ss_ext_sales_price#7))#37, avg(UnscaledValue(ss_ext_wholesale_cost#8))#38, sum(UnscaledValue(ss_ext_wholesale_cost#8))#39] -Results [4]: [avg(ss_quantity#5)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledValue(ss_ext_sales_price#7))#37 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#41, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#38 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#42, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#39,17,2) AS sum(ss_ext_wholesale_cost)#43] +Aggregate Attributes [4]: [avg(ss_quantity#5)#32, avg(UnscaledValue(ss_ext_sales_price#7))#33, avg(UnscaledValue(ss_ext_wholesale_cost#8))#34, sum(UnscaledValue(ss_ext_wholesale_cost#8))#35] +Results [4]: [avg(ss_quantity#5)#32 AS avg(ss_quantity)#36, cast((avg(UnscaledValue(ss_ext_sales_price#7))#33 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#37, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#34 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#38, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#35,17,2) AS sum(ss_ext_wholesale_cost)#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/extended.txt new file mode 100644 index 0000000000..1580f2d3cb --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/extended.txt @@ -0,0 +1,36 @@ + HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(avg(ss_quantity#1)#2 AS avg(ss_quantity)#3, cast((avg(UnscaledValue(ss_ext_sales_price#4))#5 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#6, cast((avg(UnscaledValue(ss_ext_wholesale_cost#7))#8 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#9, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#7))#10,17,2) AS sum(ss_ext_wholesale_cost)#11)] ++- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_demographics + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.household_demographics + +Comet accelerated 32 out of 33 eligible operators (96%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/simplified.txt index 9905fa96b4..e20e565fea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/simplified.txt @@ -1,37 +1,36 @@ -WholeStageCodegen (2) +WholeStageCodegen (1) HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] - CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 - CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum] + CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] + CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 + CometFilter [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/explain.txt index 61b469888b..ff9a77f0ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/explain.txt @@ -1,121 +1,172 @@ == Physical Plan == -TakeOrderedAndProject (72) -+- * HashAggregate (71) - +- Exchange (70) - +- * HashAggregate (69) - +- * Expand (68) - +- Union (67) - :- * Project (58) - : +- * Filter (57) - : +- * HashAggregate (56) - : +- Exchange (55) - : +- * HashAggregate (54) - : +- * ColumnarToRow (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) - : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometHashAggregate (32) - : : : : +- CometExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - :- * Project (62) - : +- * Filter (61) - : +- * HashAggregate (60) - : +- ReusedExchange (59) - +- * Project (66) - +- * Filter (65) - +- * HashAggregate (64) - +- ReusedExchange (63) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (102) ++- CometTakeOrderedAndProject (101) + +- CometHashAggregate (100) + +- CometExchange (99) + +- CometHashAggregate (98) + +- CometExpand (97) + +- CometUnion (96) + :- CometProject (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + :- CometProject (80) + : +- CometFilter (79) + : +- CometHashAggregate (78) + : +- CometExchange (77) + : +- CometHashAggregate (76) + : +- CometProject (75) + : +- CometBroadcastHashJoin (74) + : :- CometProject (72) + : : +- CometBroadcastHashJoin (71) + : : :- CometBroadcastHashJoin (69) + : : : :- CometFilter (67) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (66) + : : : +- ReusedExchange (68) + : : +- ReusedExchange (70) + : +- ReusedExchange (73) + +- CometProject (95) + +- CometFilter (94) + +- CometHashAggregate (93) + +- CometExchange (92) + +- CometHashAggregate (91) + +- CometProject (90) + +- CometBroadcastHashJoin (89) + :- CometProject (87) + : +- CometBroadcastHashJoin (86) + : :- CometBroadcastHashJoin (84) + : : :- CometFilter (82) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (85) + +- ReusedExchange (88) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometNativeScan: `spark_catalog`.`default`.`item` +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometNativeScan: `spark_catalog`.`default`.`item` +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] @@ -134,9 +185,12 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [d_date_sk#21, d_year#22] @@ -207,281 +261,420 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] -(34) CometBroadcastHashJoin +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(35) CometBroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [brand_id#24, class_id#25, category_id#26] -(36) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Right output [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(37) CometProject +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] -(39) CometBroadcastHashJoin +(47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(48) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : isnotnull(i_item_sk#31) +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(45) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight + +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] + +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) + +(57) CometProject +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] + +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35, d_year#36, d_moy#37] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -(48) CometFilter -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 11)) AND isnotnull(d_date_sk#35)) +(64) CometFilter +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) -(49) CometProject -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35], [d_date_sk#35] +(65) CometProject +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] +(66) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight +(67) CometFilter +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] +(68) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#57] -(53) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] +(69) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#57] +Arguments: [cs_item_sk#53], [ss_item_sk#57], LeftSemi, BuildRight -(54) HashAggregate [codegen id : 1] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#38, isEmpty#39, count#40] -Results [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#41, isEmpty#42, count#43] +(70) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] -(55) Exchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#41, isEmpty#42, count#43] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(71) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_item_sk#53], [i_item_sk#58], Inner, BuildRight -(56) HashAggregate [codegen id : 2] -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#41, isEmpty#42, count#43] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#44, count(1)#45] -Results [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#44 AS sales#46, count(1)#45 AS number_sales#47] - -(57) Filter [codegen id : 2] -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#46, number_sales#47] -Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) - -(58) Project [codegen id : 2] -Output [6]: [sales#46, number_sales#47, store AS channel#50, i_brand_id#32 AS i_brand_id#51, i_class_id#33 AS i_class_id#52, i_category_id#34 AS i_category_id#53] -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#46, number_sales#47] - -(59) ReusedExchange [Reuses operator id: 55] -Output [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#57, isEmpty#58, count#59] - -(60) HashAggregate [codegen id : 4] -Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#57, isEmpty#58, count#59] -Keys [3]: [i_brand_id#54, i_class_id#55, i_category_id#56] -Functions [2]: [sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61))#62, count(1)#63] -Results [5]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61))#62 AS sales#64, count(1)#63 AS number_sales#65] - -(61) Filter [codegen id : 4] -Input [5]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#64, number_sales#65] -Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) - -(62) Project [codegen id : 4] -Output [6]: [sales#64, number_sales#65, catalog AS channel#66, i_brand_id#54, i_class_id#55, i_category_id#56] -Input [5]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#64, number_sales#65] - -(63) ReusedExchange [Reuses operator id: 55] -Output [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#70, isEmpty#71, count#72] - -(64) HashAggregate [codegen id : 6] -Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#70, isEmpty#71, count#72] -Keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] -Functions [2]: [sum((cast(ws_quantity#73 as decimal(10,0)) * ws_list_price#74)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#73 as decimal(10,0)) * ws_list_price#74))#75, count(1)#76] -Results [5]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum((cast(ws_quantity#73 as decimal(10,0)) * ws_list_price#74))#75 AS sales#77, count(1)#76 AS number_sales#78] - -(65) Filter [codegen id : 6] -Input [5]: [i_brand_id#67, i_class_id#68, i_category_id#69, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) - -(66) Project [codegen id : 6] -Output [6]: [sales#77, number_sales#78, web AS channel#79, i_brand_id#67, i_class_id#68, i_category_id#69] -Input [5]: [i_brand_id#67, i_class_id#68, i_category_id#69, sales#77, number_sales#78] - -(67) Union - -(68) Expand [codegen id : 7] -Input [6]: [sales#46, number_sales#47, channel#50, i_brand_id#51, i_class_id#52, i_category_id#53] -Arguments: [[sales#46, number_sales#47, channel#50, i_brand_id#51, i_class_id#52, i_category_id#53, 0], [sales#46, number_sales#47, channel#50, i_brand_id#51, i_class_id#52, null, 1], [sales#46, number_sales#47, channel#50, i_brand_id#51, null, null, 3], [sales#46, number_sales#47, channel#50, null, null, null, 7], [sales#46, number_sales#47, null, null, null, null, 15]], [sales#46, number_sales#47, channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] - -(69) HashAggregate [codegen id : 7] -Input [7]: [sales#46, number_sales#47, channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] -Keys [5]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] -Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] -Aggregate Attributes [3]: [sum#85, isEmpty#86, sum#87] -Results [8]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, sum#88, isEmpty#89, sum#90] - -(70) Exchange -Input [8]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, sum#88, isEmpty#89, sum#90] -Arguments: hashpartitioning(channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(71) HashAggregate [codegen id : 8] -Input [8]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, sum#88, isEmpty#89, sum#90] -Keys [5]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] -Functions [2]: [sum(sales#46), sum(number_sales#47)] -Aggregate Attributes [2]: [sum(sales#46)#91, sum(number_sales#47)#92] -Results [6]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, sum(sales#46)#91 AS sum(sales)#93, sum(number_sales#47)#92 AS sum(number_sales)#94] - -(72) TakeOrderedAndProject -Input [6]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, sum(sales)#93, sum(number_sales)#94] -Arguments: 100, [channel#80 ASC NULLS FIRST, i_brand_id#81 ASC NULLS FIRST, i_class_id#82 ASC NULLS FIRST, i_category_id#83 ASC NULLS FIRST], [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, sum(sales)#93, sum(number_sales)#94] +(72) CometProject +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] + +(73) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#62] + +(74) CometBroadcastHashJoin +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] +Right output [1]: [d_date_sk#62] +Arguments: [cs_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(75) CometProject +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61, d_date_sk#62] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] + +(76) CometHashAggregate +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] + +(77) CometExchange +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#59, i_class_id#60, i_category_id#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(78) CometHashAggregate +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] + +(79) CometFilter +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(80) CometProject +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Arguments: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61], [sales#66, number_sales#67, catalog AS channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] + +(81) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(82) CometFilter +Input [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Condition : isnotnull(ws_item_sk#69) + +(83) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#73] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [1]: [ss_item_sk#73] +Arguments: [ws_item_sk#69], [ss_item_sk#73], LeftSemi, BuildRight + +(85) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] + +(86) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_item_sk#69], [i_item_sk#74], Inner, BuildRight + +(87) CometProject +Input [8]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] + +(88) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#78] + +(89) CometBroadcastHashJoin +Left output [6]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight + +(90) CometProject +Input [7]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, d_date_sk#78] +Arguments: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] + +(91) CometHashAggregate +Input [5]: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [partial_sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), partial_count(1)] + +(92) CometExchange +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Arguments: hashpartitioning(i_brand_id#75, i_class_id#76, i_category_id#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(93) CometHashAggregate +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), count(1)] + +(94) CometFilter +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Condition : (isnotnull(sales#82) AND (cast(sales#82 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(95) CometProject +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Arguments: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77], [sales#82, number_sales#83, web AS channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] + +(96) CometUnion +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] +Child 2 Input [6]: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] + +(97) CometExpand +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] + +(98) CometHashAggregate +Input [7]: [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] + +(99) CometExchange +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(100) CometHashAggregate +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [sum(sales#45), sum(number_sales#46)] + +(101) CometTakeOrderedAndProject +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#85 ASC NULLS FIRST,i_brand_id#86 ASC NULLS FIRST,i_class_id#87 ASC NULLS FIRST,i_category_id#88 ASC NULLS FIRST], output=[channel#85,i_brand_id#86,i_class_id#87,i_category_id#88,sum(sales)#93,sum(number_sales)#94]), [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94], 100, 0, [channel#85 ASC NULLS FIRST, i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST], [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] + +(102) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] ===== Subqueries ===== -Subquery:1 Hosting operator id = 57 Hosting Expression = Subquery scalar-subquery#48, [id=#49] -* HashAggregate (89) -+- Exchange (88) - +- * HashAggregate (87) - +- * ColumnarToRow (86) - +- CometUnion (85) - :- CometProject (76) - : +- CometBroadcastHashJoin (75) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (73) - : +- ReusedExchange (74) - :- CometProject (80) - : +- CometBroadcastHashJoin (79) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (77) - : +- ReusedExchange (78) - +- CometProject (84) - +- CometBroadcastHashJoin (83) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (81) - +- ReusedExchange (82) - - -(73) CometNativeScan: `spark_catalog`.`default`.`store_sales` +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* CometColumnarToRow (119) ++- CometHashAggregate (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometUnion (115) + :- CometProject (106) + : +- CometBroadcastHashJoin (105) + : :- CometNativeScan parquet spark_catalog.default.store_sales (103) + : +- ReusedExchange (104) + :- CometProject (110) + : +- CometBroadcastHashJoin (109) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (107) + : +- ReusedExchange (108) + +- CometProject (114) + +- CometBroadcastHashJoin (113) + :- CometNativeScan parquet spark_catalog.default.web_sales (111) + +- ReusedExchange (112) + + +(103) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] -Arguments: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#97)] +ReadSchema: struct -(74) ReusedExchange [Reuses operator id: 19] +(104) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#98] -(75) CometBroadcastHashJoin +(105) CometBroadcastHashJoin Left output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] Right output [1]: [d_date_sk#98] Arguments: [ss_sold_date_sk#97], [d_date_sk#98], Inner, BuildRight -(76) CometProject +(106) CometProject Input [4]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97, d_date_sk#98] Arguments: [quantity#99, list_price#100], [ss_quantity#95 AS quantity#99, ss_list_price#96 AS list_price#100] -(77) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(107) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] -Arguments: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#103)] +ReadSchema: struct -(78) ReusedExchange [Reuses operator id: 19] +(108) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#104] -(79) CometBroadcastHashJoin +(109) CometBroadcastHashJoin Left output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] Right output [1]: [d_date_sk#104] Arguments: [cs_sold_date_sk#103], [d_date_sk#104], Inner, BuildRight -(80) CometProject +(110) CometProject Input [4]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103, d_date_sk#104] Arguments: [quantity#105, list_price#106], [cs_quantity#101 AS quantity#105, cs_list_price#102 AS list_price#106] -(81) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(111) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] -Arguments: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#109)] +ReadSchema: struct -(82) ReusedExchange [Reuses operator id: 19] +(112) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#110] -(83) CometBroadcastHashJoin +(113) CometBroadcastHashJoin Left output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] Right output [1]: [d_date_sk#110] Arguments: [ws_sold_date_sk#109], [d_date_sk#110], Inner, BuildRight -(84) CometProject +(114) CometProject Input [4]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109, d_date_sk#110] Arguments: [quantity#111, list_price#112], [ws_quantity#107 AS quantity#111, ws_list_price#108 AS list_price#112] -(85) CometUnion +(115) CometUnion Child 0 Input [2]: [quantity#99, list_price#100] Child 1 Input [2]: [quantity#105, list_price#106] Child 2 Input [2]: [quantity#111, list_price#112] -(86) ColumnarToRow [codegen id : 1] -Input [2]: [quantity#99, list_price#100] - -(87) HashAggregate [codegen id : 1] +(116) CometHashAggregate Input [2]: [quantity#99, list_price#100] Keys: [] Functions [1]: [partial_avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] -Aggregate Attributes [2]: [sum#113, count#114] -Results [2]: [sum#115, count#116] -(88) Exchange -Input [2]: [sum#115, count#116] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(117) CometExchange +Input [2]: [sum#113, count#114] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(89) HashAggregate [codegen id : 2] -Input [2]: [sum#115, count#116] +(118) CometHashAggregate +Input [2]: [sum#113, count#114] Keys: [] Functions [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] -Aggregate Attributes [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))#117] -Results [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))#117 AS average_sales#118] -Subquery:2 Hosting operator id = 61 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] +(119) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#115] + +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] -Subquery:3 Hosting operator id = 65 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/extended.txt new file mode 100644 index 0000000000..302366bdb6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/extended.txt @@ -0,0 +1,410 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometUnion + :- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 403 out of 406 eligible operators (99%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/simplified.txt index a9e4ee92e1..208935d5f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/simplified.txt @@ -1,108 +1,126 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - WholeStageCodegen (8) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - InputAdapter - Union - WholeStageCodegen (2) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (1) - HashAggregate [quantity,list_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #9 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #9 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [d_date_sk] #9 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] + CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] + CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (1) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometBroadcastExchange [ss_item_sk] #3 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #13 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #3 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #5 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #9 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk] #9 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #7 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (4) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 - WholeStageCodegen (6) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #9 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #15 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/explain.txt index 1492cbc85d..ead8ef3316 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/explain.txt @@ -1,40 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (23) -+- * HashAggregate (22) - +- Exchange (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) +TakeOrderedAndProject (24) ++- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - +- CometBroadcastExchange (16) - +- CometProject (15) - +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.customer (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + +- CometBroadcastExchange (17) + +- CometProject (16) + +- CometFilter (15) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`customer` +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [c_customer_sk#4, c_current_addr_sk#5] @@ -53,74 +61,82 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Condition : isnotnull(ca_address_sk#6) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(11) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -Right output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight - -(12) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9, d_year#10, d_qoy#11] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#7, 2)) AS ca_state#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#8, 10)) AS ca_zip#10] -(14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] -(15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(17) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight - -(18) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] -Arguments: [cs_sales_price#2, ca_zip#8], [cs_sales_price#2, ca_zip#8] - -(19) ColumnarToRow [codegen id : 1] -Input [2]: [cs_sales_price#2, ca_zip#8] - -(20) HashAggregate [codegen id : 1] -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] +(12) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight + +(13) CometProject +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) + +(16) CometProject +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] + +(18) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight + +(19) CometProject +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] + +(20) CometHashAggregate +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [ca_zip#8, sum#13] -(21) Exchange -Input [2]: [ca_zip#8, sum#13] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(21) CometExchange +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [2]: [ca_zip#10, sum#14] -(22) HashAggregate [codegen id : 2] -Input [2]: [ca_zip#8, sum#13] -Keys [1]: [ca_zip#8] +(23) HashAggregate [codegen id : 1] +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#14] -Results [2]: [ca_zip#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#14,17,2) AS sum(cs_sales_price)#15] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#15] +Results [2]: [ca_zip#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#15,17,2) AS sum(cs_sales_price)#16] -(23) TakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#15] -Arguments: 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#15] +(24) TakeOrderedAndProject +Input [2]: [ca_zip#10, sum(cs_sales_price)#16] +Arguments: 100, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/extended.txt new file mode 100644 index 0000000000..084bb58f20 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/extended.txt @@ -0,0 +1,26 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_zip#1, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#3,17,2) AS sum(cs_sales_price)#4)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 21 out of 23 eligible operators (91%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/simplified.txt index 366d77ff30..b808324bb7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/simplified.txt @@ -1,27 +1,26 @@ TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum] - InputAdapter - Exchange [ca_zip] #1 - WholeStageCodegen (1) - HashAggregate [ca_zip,cs_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_sales_price,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] - CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] - CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 - CometFilter [ca_address_sk,ca_state,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometColumnarToRow + InputAdapter + CometExchange [ca_zip] #1 + CometHashAggregate [cs_sales_price] [ca_zip,sum] + CometProject [cs_sales_price,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] + CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] + CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] + CometFilter [ca_address_sk,ca_state,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/explain.txt index 4552905a94..eae071667f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/explain.txt @@ -1,49 +1,53 @@ == Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * HashAggregate (36) - +- * ColumnarToRow (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`call_center` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +* HashAggregate (41) ++- * CometColumnarToRow (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometNativeScan parquet spark_catalog.default.call_center (29) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] @@ -61,9 +65,11 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(6) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] -Arguments: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +ReadSchema: struct (7) CometProject Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] @@ -86,9 +92,11 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(12) CometNativeScan parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] -Arguments: [cr_order_number#12, cr_returned_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +ReadSchema: struct (13) CometProject Input [2]: [cr_order_number#12, cr_returned_date_sk#13] @@ -107,9 +115,12 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct (18) CometFilter Input [2]: [d_date_sk#14, d_date#15] @@ -132,13 +143,16 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct (24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) +Condition : ((isnotnull(ca_state#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#17, 2)) = GA)) AND isnotnull(ca_address_sk#16)) (25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] @@ -157,9 +171,12 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometNativeScan: `spark_catalog`.`default`.`call_center` +(29) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] -Arguments: [cc_call_center_sk#18, cc_county#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)] +ReadSchema: struct (30) CometFilter Input [2]: [cc_call_center_sk#18, cc_county#19] @@ -182,38 +199,39 @@ Arguments: [cs_call_center_sk#3], [cc_call_center_sk#18], Inner, BuildRight Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#18] Arguments: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(35) ColumnarToRow [codegen id : 1] -Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(36) HashAggregate [codegen id : 1] +(35) CometHashAggregate Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Keys [1]: [cs_order_number#5] Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] + +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] (37) HashAggregate [codegen id : 1] -Input [3]: [cs_order_number#5, sum#22, sum#23] +Input [3]: [cs_order_number#5, sum#20, sum#21] Keys [1]: [cs_order_number#5] Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23] +Results [3]: [cs_order_number#5, sum#20, sum#21] (38) HashAggregate [codegen id : 1] -Input [3]: [cs_order_number#5, sum#22, sum#23] +Input [3]: [cs_order_number#5, sum#20, sum#21] Keys: [] Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] + +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(39) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarToRow [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] -(40) HashAggregate [codegen id : 2] -Input [3]: [sum#22, sum#23, count#25] +(41) HashAggregate [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#21,17,2) AS total net profit #28] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] +Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#23,17,2) AS total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/extended.txt new file mode 100644 index 0000000000..dd9fdedfd0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/extended.txt @@ -0,0 +1,43 @@ + HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(count(cs_order_number#1)#2 AS order count #3, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#4))#5,17,2) AS total shipping cost #6, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#8,17,2) AS total net profit #9)] ++- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 36 out of 39 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/simplified.txt index 4cd5e96597..006d2d1956 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/simplified.txt @@ -1,44 +1,45 @@ WholeStageCodegen (2) HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + CometColumnarToRow InputAdapter - CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] - CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] - CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometSort [cs_warehouse_sk,cs_order_number] - CometExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - CometSort [cr_order_number] - CometExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_order_number,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [cc_call_center_sk] #7 - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_county] + CometHashAggregate [cs_ext_ship_cost,cs_net_profit] [cs_order_number,sum,sum] + CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] + CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] + CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cc_call_center_sk] #7 + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/explain.txt index 022dab9eb9..e1432039dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) : :- CometProject (27) : : +- CometBroadcastHashJoin (26) : : :- CometProject (24) @@ -19,41 +19,51 @@ TakeOrderedAndProject (42) : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`item` (33) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.item (34) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] @@ -72,9 +82,13 @@ Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] @@ -93,13 +107,16 @@ Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_quarter_name#17] -Arguments: [d_date_sk#16, d_quarter_name#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_quarter_name), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) +Condition : ((isnotnull(d_quarter_name#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#17, 6)) = 2001Q1)) AND isnotnull(d_date_sk#16)) (15) CometProject Input [2]: [d_date_sk#16, d_quarter_name#17] @@ -118,13 +135,16 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_quarter_name#19] -Arguments: [d_date_sk#18, d_quarter_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct (20) CometFilter Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#19, 6)) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) (21) CometProject Input [2]: [d_date_sk#18, d_quarter_name#19] @@ -155,70 +175,80 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(28) CometNativeScan: `spark_catalog`.`default`.`store` +(28) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (29) CometFilter Input [2]: [s_store_sk#21, s_state#22] Condition : isnotnull(s_store_sk#21) -(30) CometBroadcastExchange +(30) CometProject Input [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#22, 2)) AS s_state#23] + +(31) CometBroadcastExchange +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] -(31) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [2]: [s_store_sk#21, s_state#22] +Right output [2]: [s_store_sk#21, s_state#23] Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight -(32) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] - -(33) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] - -(34) CometFilter -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) - -(35) CometBroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] - -(36) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Right output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_item_sk#1], [i_item_sk#23], Inner, BuildRight - -(37) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] - -(38) ColumnarToRow [codegen id : 1] -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] - -(39) HashAggregate [codegen id : 1] -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(33) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] + +(34) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(35) CometFilter +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) + +(36) CometProject +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#25, 16)) AS i_item_id#27, i_item_desc#26] + +(37) CometBroadcastExchange +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] + +(38) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight + +(39) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] + +(40) CometHashAggregate +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [18]: [count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Results [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -(40) Exchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(41) CometExchange +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(41) HashAggregate [codegen id : 2] -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(42) CometHashAggregate +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [9]: [count(ss_quantity#5)#62, avg(ss_quantity#5)#63, stddev_samp(cast(ss_quantity#5 as double))#64, count(sr_return_quantity#10)#65, avg(sr_return_quantity#10)#66, stddev_samp(cast(sr_return_quantity#10 as double))#67, count(cs_quantity#14)#68, avg(cs_quantity#14)#69, stddev_samp(cast(cs_quantity#14 as double))#70] -Results [15]: [i_item_id#24, i_item_desc#25, s_state#22, count(ss_quantity#5)#62 AS store_sales_quantitycount#71, avg(ss_quantity#5)#63 AS store_sales_quantityave#72, stddev_samp(cast(ss_quantity#5 as double))#64 AS store_sales_quantitystdev#73, (stddev_samp(cast(ss_quantity#5 as double))#64 / avg(ss_quantity#5)#63) AS store_sales_quantitycov#74, count(sr_return_quantity#10)#65 AS as_store_returns_quantitycount#75, avg(sr_return_quantity#10)#66 AS as_store_returns_quantityave#76, stddev_samp(cast(sr_return_quantity#10 as double))#67 AS as_store_returns_quantitystdev#77, (stddev_samp(cast(sr_return_quantity#10 as double))#67 / avg(sr_return_quantity#10)#66) AS store_returns_quantitycov#78, count(cs_quantity#14)#68 AS catalog_sales_quantitycount#79, avg(cs_quantity#14)#69 AS catalog_sales_quantityave#80, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitystdev#81, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitycov#82] -(42) TakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] -Arguments: 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] +(43) CometTakeOrderedAndProject +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] + +(44) CometColumnarToRow [codegen id : 1] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/simplified.txt index b79443c9a6..97c37ed5a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/simplified.txt @@ -1,46 +1,46 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - InputAdapter - Exchange [i_item_id,i_item_desc,s_state] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] - CometBroadcastExchange [d_date_sk] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] + CometHashAggregate [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] + CometExchange [i_item_id,i_item_desc,s_state] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [s_store_sk,s_state] #6 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_state] #6 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/explain.txt index bc0faccc0a..dbe4513617 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -TakeOrderedAndProject (41) -+- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * ColumnarToRow (37) - +- CometExpand (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (43) ++- CometTakeOrderedAndProject (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometExpand (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) : : :- CometProject (19) : : : +- CometBroadcastHashJoin (18) : : : :- CometProject (14) @@ -18,45 +18,54 @@ TakeOrderedAndProject (41) : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (15) - : : +- CometBroadcastExchange (22) - : : +- CometFilter (21) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - : +- CometBroadcastExchange (28) - : +- CometProject (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - +- CometBroadcastExchange (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`item` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.date_dim (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.item (32) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Arguments: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) +Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = Unknown )) AND isnotnull(cd_demo_sk#10)) (5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] @@ -75,9 +84,12 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(9) CometNativeScan: `spark_catalog`.`default`.`customer` +(9) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (10) CometFilter Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] @@ -100,9 +112,12 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(15) CometNativeScan parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [1]: [cd_demo_sk#19] @@ -121,99 +136,112 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) -(22) CometBroadcastExchange +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24, d_year#25] - -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) - -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] - -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight - -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(31) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] - -(36) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] - -(37) ColumnarToRow [codegen id : 1] -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] - -(38) HashAggregate [codegen id : 1] -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(26) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) + +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight + +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] + +(38) CometExpand +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] + +(39) CometHashAggregate +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [14]: [sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Results [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -(39) Exchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(40) CometExchange +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(40) HashAggregate [codegen id : 2] -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(41) CometHashAggregate +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#61, avg(cast(cs_list_price#5 as decimal(12,2)))#62, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63, avg(cast(cs_sales_price#6 as decimal(12,2)))#64, avg(cast(cs_net_profit#8 as decimal(12,2)))#65, avg(cast(c_birth_year#18 as decimal(12,2)))#66, avg(cast(cd_dep_count#13 as decimal(12,2)))#67] -Results [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, avg(cast(cs_quantity#4 as decimal(12,2)))#61 AS agg1#68, avg(cast(cs_list_price#5 as decimal(12,2)))#62 AS agg2#69, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63 AS agg3#70, avg(cast(cs_sales_price#6 as decimal(12,2)))#64 AS agg4#71, avg(cast(cs_net_profit#8 as decimal(12,2)))#65 AS agg5#72, avg(cast(c_birth_year#18 as decimal(12,2)))#66 AS agg6#73, avg(cast(cd_dep_count#13 as decimal(12,2)))#67 AS agg7#74] -(41) TakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] -Arguments: 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] +(42) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] + +(43) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/extended.txt new file mode 100644 index 0000000000..63cf56de78 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/simplified.txt index 87e5a9e480..4507330941 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/simplified.txt @@ -1,45 +1,45 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - WholeStageCodegen (2) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #4 - CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] + CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/explain.txt index 0f1446bb56..d207066da3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) +TakeOrderedAndProject (36) ++- * HashAggregate (35) + +- * CometColumnarToRow (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (19) : : +- CometBroadcastHashJoin (18) : : :- CometProject (14) @@ -16,26 +16,33 @@ TakeOrderedAndProject (32) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + : : : +- CometNativeScan parquet spark_catalog.default.item (9) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - +- ReusedExchange (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` + : : +- CometNativeScan parquet spark_catalog.default.customer (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.store (26) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -45,9 +52,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] @@ -66,9 +77,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] @@ -76,97 +90,121 @@ Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnul (11) CometProject Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [i_item_sk#9, i_brand_id#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#11, 50)) AS i_brand#15, i_manufact_id#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#13, 50)) AS i_manufact#16] (12) CometBroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(15) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] +(15) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (16) CometFilter -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) (17) CometBroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [c_customer_sk#17, c_current_addr_sk#18] (18) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Right output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_customer_sk#5], [c_customer_sk#15], Inner, BuildRight +Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight (19) CometProject -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] +Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] +(20) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#19, ca_zip#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (21) CometFilter -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) - -(22) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Right output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight - -(24) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] - -(25) ReusedExchange [Reuses operator id: 22] -Output [2]: [s_store_sk#19, s_zip#20] - -(26) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Right output [2]: [s_store_sk#19, s_zip#20] -Arguments: [ss_store_sk#6], [s_store_sk#19], Inner, NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)), BuildRight - -(27) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] -Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(28) ColumnarToRow [codegen id : 1] -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(29) HashAggregate [codegen id : 1] -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +Input [2]: [ca_address_sk#19, ca_zip#20] +Condition : (isnotnull(ca_address_sk#19) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#20, 10)))) + +(22) CometProject +Input [2]: [ca_address_sk#19, ca_zip#20] +Arguments: [ca_address_sk#19, ca_zip#21], [ca_address_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#20, 10)) AS ca_zip#21] + +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [ca_address_sk#19, ca_zip#21] + +(24) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] +Right output [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight + +(25) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] + +(26) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#22, s_zip#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [s_store_sk#22, s_zip#23] +Condition : (isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#23, 10))) AND isnotnull(s_store_sk#22)) + +(28) CometProject +Input [2]: [s_store_sk#22, s_zip#23] +Arguments: [s_store_sk#22, s_zip#24], [s_store_sk#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#23, 10)) AS s_zip#24] + +(29) CometBroadcastExchange +Input [2]: [s_store_sk#22, s_zip#24] +Arguments: [s_store_sk#22, s_zip#24] + +(30) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] +Right output [2]: [s_store_sk#22, s_zip#24] +Arguments: [ss_store_sk#6], [s_store_sk#22], Inner, NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5)), BuildRight + +(31) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21, s_store_sk#22, s_zip#24] +Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] + +(32) CometHashAggregate +Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum#21] -Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -(30) Exchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(33) CometExchange +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Arguments: hashpartitioning(i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] -(31) HashAggregate [codegen id : 2] -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(35) HashAggregate [codegen id : 1] +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#23] -Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#23,17,2) AS ext_price#26] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#26] +Results [5]: [i_brand_id#10 AS brand_id#27, i_brand#15 AS brand#28, i_manufact_id#12, i_manufact#16, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#26,17,2) AS ext_price#29] -(32) TakeOrderedAndProject -Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] -Arguments: 100, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] +(36) TakeOrderedAndProject +Input [5]: [brand_id#27, brand#28, i_manufact_id#12, i_manufact#16, ext_price#29] +Arguments: 100, [ext_price#29 DESC NULLS LAST, brand#28 ASC NULLS FIRST, brand_id#27 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#16 ASC NULLS FIRST], [brand_id#27, brand#28, i_manufact_id#12, i_manufact#16, ext_price#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/extended.txt new file mode 100644 index 0000000000..093e5fac16 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/extended.txt @@ -0,0 +1,38 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_brand_id#1 AS brand_id#2, i_brand#3 AS brand#4, i_manufact_id#5, i_manufact#6, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#8,17,2) AS ext_price#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 33 out of 35 eligible operators (94%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/simplified.txt index 2be817efe9..ccb7932010 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/simplified.txt @@ -1,36 +1,38 @@ TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] - CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 - CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_zip] #5 - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] - ReusedExchange [s_store_sk,s_zip] #5 + CometColumnarToRow + InputAdapter + CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] + CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 + CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_zip] #5 + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [s_store_sk,s_zip] #6 + CometProject [s_zip] [s_store_sk,s_zip] + CometFilter [s_store_sk,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/explain.txt index 4239f9f403..4f97c551ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/explain.txt @@ -1,53 +1,61 @@ == Physical Plan == -* ColumnarToRow (36) -+- CometSort (35) - +- CometColumnarExchange (34) - +- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * HashAggregate (14) - : : +- Exchange (13) - : : +- * HashAggregate (12) - : : +- * ColumnarToRow (11) - : : +- CometProject (10) - : : +- CometBroadcastHashJoin (9) +* CometColumnarToRow (38) ++- CometSort (37) + +- CometColumnarExchange (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * Project (22) + : +- * BroadcastHashJoin Inner BuildRight (21) + : :- * HashAggregate (15) + : : +- * CometColumnarToRow (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) : : :- CometUnion (5) : : : :- CometProject (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometProject (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : +- CometBroadcastExchange (8) - : : +- CometFilter (7) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (6) - : +- BroadcastExchange (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (15) - +- BroadcastExchange (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * HashAggregate (23) - : +- ReusedExchange (22) - +- BroadcastExchange (28) - +- * ColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (24) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (3) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (6) + : +- BroadcastExchange (20) + : +- * CometColumnarToRow (19) + : +- CometProject (18) + : +- CometFilter (17) + : +- CometNativeScan parquet spark_catalog.default.date_dim (16) + +- BroadcastExchange (33) + +- * Project (32) + +- * BroadcastHashJoin Inner BuildRight (31) + :- * HashAggregate (25) + : +- * CometColumnarToRow (24) + : +- ReusedExchange (23) + +- BroadcastExchange (30) + +- * CometColumnarToRow (29) + +- CometProject (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.date_dim (26) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] -Arguments: [ws_ext_sales_price#1, ws_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#2)] +ReadSchema: struct (2) CometProject Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(3) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] -Arguments: [cs_ext_sales_price#5, cs_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#6)] +ReadSchema: struct (4) CometProject Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] @@ -57,138 +65,152 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(6) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] +ReadSchema: struct (7) CometFilter Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(8) CometBroadcastExchange +(8) CometProject Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12], [d_date_sk#9, d_week_seq#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(d_day_name#11, 9)) AS d_day_name#12] -(9) CometBroadcastHashJoin +(9) CometBroadcastExchange +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12] + +(10) CometBroadcastHashJoin Left output [2]: [sold_date_sk#3, sales_price#4] -Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(10) CometProject -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [sales_price#4, d_week_seq#10, d_day_name#11], [sales_price#4, d_week_seq#10, d_day_name#11] - -(11) ColumnarToRow [codegen id : 1] -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] - -(12) HashAggregate [codegen id : 1] -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] -Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] - -(13) Exchange -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(11) CometProject +Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sales_price#4, d_week_seq#10, d_day_name#12], [sales_price#4, d_week_seq#10, d_day_name#12] -(14) HashAggregate [codegen id : 6] -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] +(12) CometHashAggregate +Input [3]: [sales_price#4, d_week_seq#10, d_day_name#12] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(15) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_week_seq#40, d_year#41] -Arguments: [d_week_seq#40, d_year#41] +(13) CometExchange +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometFilter -Input [2]: [d_week_seq#40, d_year#41] -Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2001)) AND isnotnull(d_week_seq#40)) +(14) CometColumnarToRow [codegen id : 4] +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] -(17) CometProject -Input [2]: [d_week_seq#40, d_year#41] -Arguments: [d_week_seq#40], [d_week_seq#40] - -(18) ColumnarToRow [codegen id : 2] -Input [1]: [d_week_seq#40] - -(19) BroadcastExchange -Input [1]: [d_week_seq#40] +(15) HashAggregate [codegen id : 4] +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [1]: [d_week_seq#10] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))#26] +Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END))#20,17,2) AS sun_sales#27, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END))#21,17,2) AS mon_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END))#22,17,2) AS tue_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END))#23,17,2) AS wed_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END))#24,17,2) AS thu_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END))#25,17,2) AS fri_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))#26,17,2) AS sat_sales#33] + +(16) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#34, d_year#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [d_week_seq#34, d_year#35] +Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2001)) AND isnotnull(d_week_seq#34)) + +(18) CometProject +Input [2]: [d_week_seq#34, d_year#35] +Arguments: [d_week_seq#34], [d_week_seq#34] + +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#34] + +(20) BroadcastExchange +Input [1]: [d_week_seq#34] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(20) BroadcastHashJoin [codegen id : 6] +(21) BroadcastHashJoin [codegen id : 4] Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#40] +Right keys [1]: [d_week_seq#34] Join type: Inner Join condition: None -(21) Project [codegen id : 6] -Output [8]: [d_week_seq#10 AS d_week_seq1#42, sun_sales#33 AS sun_sales1#43, mon_sales#34 AS mon_sales1#44, tue_sales#35 AS tue_sales1#45, wed_sales#36 AS wed_sales1#46, thu_sales#37 AS thu_sales1#47, fri_sales#38 AS fri_sales1#48, sat_sales#39 AS sat_sales1#49] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#40] +(22) Project [codegen id : 4] +Output [8]: [d_week_seq#10 AS d_week_seq1#36, sun_sales#27 AS sun_sales1#37, mon_sales#28 AS mon_sales1#38, tue_sales#29 AS tue_sales1#39, wed_sales#30 AS wed_sales1#40, thu_sales#31 AS thu_sales1#41, fri_sales#32 AS fri_sales1#42, sat_sales#33 AS sat_sales1#43] +Input [9]: [d_week_seq#10, sun_sales#27, mon_sales#28, tue_sales#29, wed_sales#30, thu_sales#31, fri_sales#32, sat_sales#33, d_week_seq#34] + +(23) ReusedExchange [Reuses operator id: 13] +Output [8]: [d_week_seq#44, sum#45, sum#46, sum#47, sum#48, sum#49, sum#50, sum#51] -(22) ReusedExchange [Reuses operator id: 13] -Output [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] +(24) CometColumnarToRow [codegen id : 3] +Input [8]: [d_week_seq#44, sum#45, sum#46, sum#47, sum#48, sum#49, sum#50, sum#51] -(23) HashAggregate [codegen id : 5] -Input [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] -Keys [1]: [d_week_seq#50] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32] -Results [8]: [d_week_seq#50, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26,17,2) AS sun_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27,17,2) AS mon_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28,17,2) AS tue_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29,17,2) AS wed_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30,17,2) AS thu_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31,17,2) AS fri_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32,17,2) AS sat_sales#66] +(25) HashAggregate [codegen id : 3] +Input [8]: [d_week_seq#44, sum#45, sum#46, sum#47, sum#48, sum#49, sum#50, sum#51] +Keys [1]: [d_week_seq#44] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#52 = Sunday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Monday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Tuesday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Wednesday) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Thursday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Friday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Saturday ) THEN sales_price#53 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#52 = Sunday ) THEN sales_price#53 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Monday ) THEN sales_price#53 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Tuesday ) THEN sales_price#53 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Wednesday) THEN sales_price#53 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Thursday ) THEN sales_price#53 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Friday ) THEN sales_price#53 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Saturday ) THEN sales_price#53 END))#26] +Results [8]: [d_week_seq#44, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Sunday ) THEN sales_price#53 END))#20,17,2) AS sun_sales#54, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Monday ) THEN sales_price#53 END))#21,17,2) AS mon_sales#55, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Tuesday ) THEN sales_price#53 END))#22,17,2) AS tue_sales#56, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Wednesday) THEN sales_price#53 END))#23,17,2) AS wed_sales#57, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Thursday ) THEN sales_price#53 END))#24,17,2) AS thu_sales#58, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Friday ) THEN sales_price#53 END))#25,17,2) AS fri_sales#59, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Saturday ) THEN sales_price#53 END))#26,17,2) AS sat_sales#60] -(24) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_week_seq#67, d_year#68] -Arguments: [d_week_seq#67, d_year#68] +(26) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#61, d_year#62] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] +ReadSchema: struct -(25) CometFilter -Input [2]: [d_week_seq#67, d_year#68] -Condition : ((isnotnull(d_year#68) AND (d_year#68 = 2002)) AND isnotnull(d_week_seq#67)) +(27) CometFilter +Input [2]: [d_week_seq#61, d_year#62] +Condition : ((isnotnull(d_year#62) AND (d_year#62 = 2002)) AND isnotnull(d_week_seq#61)) -(26) CometProject -Input [2]: [d_week_seq#67, d_year#68] -Arguments: [d_week_seq#67], [d_week_seq#67] +(28) CometProject +Input [2]: [d_week_seq#61, d_year#62] +Arguments: [d_week_seq#61], [d_week_seq#61] -(27) ColumnarToRow [codegen id : 4] -Input [1]: [d_week_seq#67] +(29) CometColumnarToRow [codegen id : 2] +Input [1]: [d_week_seq#61] -(28) BroadcastExchange -Input [1]: [d_week_seq#67] +(30) BroadcastExchange +Input [1]: [d_week_seq#61] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [d_week_seq#50] -Right keys [1]: [d_week_seq#67] +(31) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [d_week_seq#44] +Right keys [1]: [d_week_seq#61] Join type: Inner Join condition: None -(30) Project [codegen id : 5] -Output [8]: [d_week_seq#50 AS d_week_seq2#69, sun_sales#60 AS sun_sales2#70, mon_sales#61 AS mon_sales2#71, tue_sales#62 AS tue_sales2#72, wed_sales#63 AS wed_sales2#73, thu_sales#64 AS thu_sales2#74, fri_sales#65 AS fri_sales2#75, sat_sales#66 AS sat_sales2#76] -Input [9]: [d_week_seq#50, sun_sales#60, mon_sales#61, tue_sales#62, wed_sales#63, thu_sales#64, fri_sales#65, sat_sales#66, d_week_seq#67] +(32) Project [codegen id : 3] +Output [8]: [d_week_seq#44 AS d_week_seq2#63, sun_sales#54 AS sun_sales2#64, mon_sales#55 AS mon_sales2#65, tue_sales#56 AS tue_sales2#66, wed_sales#57 AS wed_sales2#67, thu_sales#58 AS thu_sales2#68, fri_sales#59 AS fri_sales2#69, sat_sales#60 AS sat_sales2#70] +Input [9]: [d_week_seq#44, sun_sales#54, mon_sales#55, tue_sales#56, wed_sales#57, thu_sales#58, fri_sales#59, sat_sales#60, d_week_seq#61] -(31) BroadcastExchange -Input [8]: [d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] +(33) BroadcastExchange +Input [8]: [d_week_seq2#63, sun_sales2#64, mon_sales2#65, tue_sales2#66, wed_sales2#67, thu_sales2#68, fri_sales2#69, sat_sales2#70] Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=4] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [d_week_seq1#42] -Right keys [1]: [(d_week_seq2#69 - 53)] +(34) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [d_week_seq1#36] +Right keys [1]: [(d_week_seq2#63 - 53)] Join type: Inner Join condition: None -(33) Project [codegen id : 6] -Output [8]: [d_week_seq1#42, round((sun_sales1#43 / sun_sales2#70), 2) AS round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1#44 / mon_sales2#71), 2) AS round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1#45 / tue_sales2#72), 2) AS round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1#46 / wed_sales2#73), 2) AS round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1#47 / thu_sales2#74), 2) AS round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1#48 / fri_sales2#75), 2) AS round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1#49 / sat_sales2#76), 2) AS round((sat_sales1 / sat_sales2), 2)#83] -Input [16]: [d_week_seq1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] +(35) Project [codegen id : 4] +Output [8]: [d_week_seq1#36, round((sun_sales1#37 / sun_sales2#64), 2) AS round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1#38 / mon_sales2#65), 2) AS round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1#39 / tue_sales2#66), 2) AS round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1#40 / wed_sales2#67), 2) AS round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1#41 / thu_sales2#68), 2) AS round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1#42 / fri_sales2#69), 2) AS round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1#43 / sat_sales2#70), 2) AS round((sat_sales1 / sat_sales2), 2)#77] +Input [16]: [d_week_seq1#36, sun_sales1#37, mon_sales1#38, tue_sales1#39, wed_sales1#40, thu_sales1#41, fri_sales1#42, sat_sales1#43, d_week_seq2#63, sun_sales2#64, mon_sales2#65, tue_sales2#66, wed_sales2#67, thu_sales2#68, fri_sales2#69, sat_sales2#70] -(34) CometColumnarExchange -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] -Arguments: rangepartitioning(d_week_seq1#42 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +(36) CometColumnarExchange +Input [8]: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77] +Arguments: rangepartitioning(d_week_seq1#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(35) CometSort -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] -Arguments: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83], [d_week_seq1#42 ASC NULLS FIRST] +(37) CometSort +Input [8]: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77] +Arguments: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77], [d_week_seq1#36 ASC NULLS FIRST] -(36) ColumnarToRow [codegen id : 7] -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] +(38) CometColumnarToRow [codegen id : 5] +Input [8]: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/extended.txt new file mode 100644 index 0000000000..0547ee471f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/extended.txt @@ -0,0 +1,52 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_week_seq#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#2 = Sunday ) THEN sales_price#3 END))#4,17,2) AS sun_sales#5, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#2 = Monday ) THEN sales_price#3 END))#6,17,2) AS mon_sales#7, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#2 = Tuesday ) THEN sales_price#3 END))#8,17,2) AS tue_sales#9, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#2 = Wednesday) THEN sales_price#3 END))#10,17,2) AS wed_sales#11, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#2 = Thursday ) THEN sales_price#3 END))#12,17,2) AS thu_sales#13, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#2 = Friday ) THEN sales_price#3 END))#14,17,2) AS fri_sales#15, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#2 = Saturday ) THEN sales_price#3 END))#16,17,2) AS sat_sales#17)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_week_seq#18, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#19 = Sunday ) THEN sales_price#20 END))#4,17,2) AS sun_sales#21, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#19 = Monday ) THEN sales_price#20 END))#6,17,2) AS mon_sales#22, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#19 = Tuesday ) THEN sales_price#20 END))#8,17,2) AS tue_sales#23, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#19 = Wednesday) THEN sales_price#20 END))#10,17,2) AS wed_sales#24, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#19 = Thursday ) THEN sales_price#20 END))#12,17,2) AS thu_sales#25, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#19 = Friday ) THEN sales_price#20 END))#14,17,2) AS fri_sales#26, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#19 = Saturday ) THEN sales_price#20 END))#16,17,2) AS sat_sales#27)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 34 out of 45 eligible operators (75%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/simplified.txt index 9137b04fe5..af58e62f55 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/simplified.txt @@ -1,51 +1,51 @@ -WholeStageCodegen (7) - ColumnarToRow +WholeStageCodegen (5) + CometColumnarToRow InputAdapter CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] CometColumnarExchange [d_week_seq1] #1 - WholeStageCodegen (6) + WholeStageCodegen (4) Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] BroadcastHashJoin [d_week_seq1,d_week_seq2] Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq] #2 - WholeStageCodegen (1) - HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] - CometUnion [sold_date_sk,sales_price] - CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ext_sales_price,ws_sold_date_sk] - CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] + CometColumnarToRow + InputAdapter + CometExchange [d_week_seq] #2 + CometHashAggregate [d_day_name,sales_price] [d_week_seq,sum,sum,sum,sum,sum,sum,sum] + CometProject [sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] + CometUnion [sold_date_sk,sales_price] + CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #4 - WholeStageCodegen (2) - ColumnarToRow + WholeStageCodegen (1) + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] InputAdapter BroadcastExchange #5 - WholeStageCodegen (5) + WholeStageCodegen (3) Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 + CometColumnarToRow + InputAdapter + ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 InputAdapter BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow + WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/explain.txt index 81b0ce5c0b..d72ed51951 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/explain.txt @@ -1,120 +1,137 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * Sort (19) - +- Exchange (18) - +- * HashAggregate (17) - +- Exchange (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) ColumnarToRow [codegen id : 1] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(15) HashAggregate [codegen id : 1] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(16) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) HashAggregate [codegen id : 2] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] + +(19) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(18) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(20) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) Sort [codegen id : 3] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(22) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] -(21) Project [codegen id : 4] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(23) Project [codegen id : 3] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(24) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/extended.txt new file mode 100644 index 0000000000..955c18b30c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/extended.txt @@ -0,0 +1,26 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_desc#1, i_category#2, i_class#3, i_current_price#4, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#5))#6,17,2) AS itemrevenue#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#5))#6,17,2) AS _w0#8, i_item_id#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 18 out of 22 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/simplified.txt index 2bc9570d10..01f11aac85 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/simplified.txt @@ -1,30 +1,30 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] InputAdapter Window [_w0,i_class] - WholeStageCodegen (3) - Sort [i_class] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/explain.txt index 180d4fd802..13f8755611 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Filter (23) - +- * HashAggregate (22) - +- Exchange (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometFilter (22) + +- CometHashAggregate (21) + +- CometExchange (20) + +- CometHashAggregate (19) +- CometProject (18) +- CometBroadcastHashJoin (17) :- CometProject (13) @@ -12,30 +12,37 @@ TakeOrderedAndProject (24) : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (8) + : +- CometNativeScan parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [w_warehouse_sk#5, w_warehouse_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] @@ -54,9 +61,12 @@ Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -(8) CometNativeScan: `spark_catalog`.`default`.`item` +(8) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8, i_current_price#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] @@ -64,68 +74,67 @@ Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (10) CometProject Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8], [i_item_sk#7, i_item_id#8] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#8, 16)) AS i_item_id#10] (11) CometBroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: [i_item_sk#7, i_item_id#8] +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] (12) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -Right output [2]: [i_item_sk#7, i_item_id#8] +Right output [2]: [i_item_sk#7, i_item_id#10] Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_date#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) (16) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] (17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [inv_date_sk#4], [d_date_sk#10], Inner, BuildRight +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight (18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] - -(19) ColumnarToRow [codegen id : 1] -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] - -(20) HashAggregate [codegen id : 1] -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum#12, sum#13] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] - -(21) Exchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 2] -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16 AS inv_before#18, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17 AS inv_after#19] - -(23) Filter [codegen id : 2] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Condition : (CASE WHEN (inv_before#18 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#19 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#18 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#18 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#19 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#18 as double)))))) <= 1.5) END) - -(24) TakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Arguments: 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] + +(19) CometHashAggregate +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(20) CometExchange +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(22) CometFilter +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) + +(23) CometTakeOrderedAndProject +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] + +(24) CometColumnarToRow [codegen id : 1] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/extended.txt new file mode 100644 index 0000000000..92628ea98d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/extended.txt @@ -0,0 +1,26 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 23 out of 23 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/simplified.txt index 4b5b8afc29..f0cc321417 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/simplified.txt @@ -1,28 +1,26 @@ -TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] - WholeStageCodegen (2) - Filter [inv_before,inv_after] - HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum] - InputAdapter - Exchange [w_warehouse_name,i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] - CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] - CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_id] #3 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #4 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] + CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] + CometHashAggregate [sum,sum] [w_warehouse_name,i_item_id,inv_before,inv_after,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] + CometExchange [w_warehouse_name,i_item_id] #1 + CometHashAggregate [d_date,inv_quantity_on_hand] [w_warehouse_name,i_item_id,sum,sum] + CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] + CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] + CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_id] #3 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #4 + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/explain.txt index c097db94ab..a5c321f899 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/explain.txt @@ -1,41 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (25) ++- CometTakeOrderedAndProject (24) + +- CometHashAggregate (23) + +- CometExchange (22) + +- CometHashAggregate (21) + +- CometExpand (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.warehouse (15) -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] @@ -58,74 +66,80 @@ Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Condition : isnotnull(i_item_sk#7) -(11) CometBroadcastExchange +(11) CometProject Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#8, 50)) AS i_brand#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#11, 50)) AS i_product_name#15] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(13) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] - -(19) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(20) ColumnarToRow [codegen id : 1] -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(21) HashAggregate [codegen id : 1] -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(15) CometNativeScan parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) + +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] + +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight + +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] + +(20) CometExpand +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] + +(21) CometHashAggregate +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#18, count#19] -Results [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -(22) Exchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(22) CometExchange +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 2] -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(23) CometHashAggregate +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#22] -Results [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, avg(inv_quantity_on_hand#3)#22 AS qoh#23] -(24) TakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] -Arguments: 100, [qoh#23 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] +(24) CometTakeOrderedAndProject +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] + +(25) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/extended.txt new file mode 100644 index 0000000000..275cecfd63 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/simplified.txt index 60fd5802d3..8d2b671164 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/simplified.txt @@ -1,28 +1,27 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (1) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] - CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #4 - CometFilter [w_warehouse_sk] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [spark_grouping_id,sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] + CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] + CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/explain.txt index 67f7d13147..8fc110239e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/explain.txt @@ -1,91 +1,98 @@ == Physical Plan == -* HashAggregate (70) -+- Exchange (69) - +- * HashAggregate (68) - +- Union (67) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- * SortMergeJoin LeftSemi (41) - : : :- * ColumnarToRow (25) - : : : +- CometSort (24) - : : : +- CometExchange (23) - : : : +- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometBroadcastExchange (20) - : : : +- CometProject (19) - : : : +- CometFilter (18) - : : : +- CometHashAggregate (17) - : : : +- CometExchange (16) - : : : +- CometHashAggregate (15) - : : : +- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometFilter (3) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (2) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- CometBroadcastExchange (12) - : : : +- CometFilter (11) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (10) - : : +- * Sort (40) - : : +- * Project (39) - : : +- * Filter (38) - : : +- * HashAggregate (37) - : : +- Exchange (36) - : : +- * HashAggregate (35) - : : +- * ColumnarToRow (34) - : : +- CometProject (33) - : : +- CometBroadcastHashJoin (32) - : : :- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) - : : +- CometBroadcastExchange (31) - : : +- CometFilter (30) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometProject (45) - : +- CometFilter (44) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (43) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * SortMergeJoin LeftSemi (62) - : :- * ColumnarToRow (56) - : : +- CometSort (55) - : : +- CometExchange (54) - : : +- CometProject (53) - : : +- CometBroadcastHashJoin (52) - : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (50) - : : +- ReusedExchange (51) - : +- * Sort (61) - : +- * Project (60) - : +- * Filter (59) - : +- * HashAggregate (58) - : +- ReusedExchange (57) - +- ReusedExchange (64) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +* CometColumnarToRow (67) ++- CometHashAggregate (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometUnion (63) + :- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (40) + : : +- CometSortMergeJoin (39) + : : :- CometSort (24) + : : : +- CometExchange (23) + : : : +- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (20) + : : : +- CometProject (19) + : : : +- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometFilter (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (2) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometNativeScan parquet spark_catalog.default.item (10) + : : +- CometSort (38) + : : +- CometProject (37) + : : +- CometFilter (36) + : : +- CometHashAggregate (35) + : : +- CometExchange (34) + : : +- CometHashAggregate (33) + : : +- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometProject (27) + : : : +- CometFilter (26) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (25) + : : +- CometBroadcastExchange (30) + : : +- CometFilter (29) + : : +- CometNativeScan parquet spark_catalog.default.customer (28) + : +- CometBroadcastExchange (44) + : +- CometProject (43) + : +- CometFilter (42) + : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + +- CometProject (62) + +- CometBroadcastHashJoin (61) + :- CometProject (59) + : +- CometSortMergeJoin (58) + : :- CometSort (52) + : : +- CometExchange (51) + : : +- CometProject (50) + : : +- CometBroadcastHashJoin (49) + : : :- CometNativeScan parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (48) + : +- CometSort (57) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometHashAggregate (54) + : +- ReusedExchange (53) + +- ReusedExchange (60) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +ReadSchema: struct -(2) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(2) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (3) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] @@ -108,9 +115,12 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(10) CometNativeScan: `spark_catalog`.`default`.`item` +(10) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (11) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] @@ -172,313 +182,303 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(25) ColumnarToRow [codegen id : 1] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(25) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct -(27) CometFilter +(26) CometFilter Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Condition : isnotnull(ss_customer_sk#17) -(28) CometProject +(27) CometProject Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(29) CometNativeScan: `spark_catalog`.`default`.`customer` +(28) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(30) CometFilter +(29) CometFilter Input [1]: [c_customer_sk#21] Condition : isnotnull(c_customer_sk#21) -(31) CometBroadcastExchange +(30) CometBroadcastExchange Input [1]: [c_customer_sk#21] Arguments: [c_customer_sk#21] -(32) CometBroadcastHashJoin +(31) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] Right output [1]: [c_customer_sk#21] Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight -(33) CometProject +(32) CometProject Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -(34) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(35) HashAggregate [codegen id : 2] +(33) CometHashAggregate Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] Keys [1]: [c_customer_sk#21] Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [2]: [sum#22, isEmpty#23] -Results [3]: [c_customer_sk#21, sum#24, isEmpty#25] -(36) Exchange -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(34) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) HashAggregate [codegen id : 3] -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] +(35) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] Keys [1]: [c_customer_sk#21] Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26] -Results [2]: [c_customer_sk#21, sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26 AS ssales#27] -(38) Filter [codegen id : 3] -Input [2]: [c_customer_sk#21, ssales#27] -Condition : (isnotnull(ssales#27) AND (cast(ssales#27 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#28, [id=#29]))) +(36) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) -(39) Project [codegen id : 3] -Output [1]: [c_customer_sk#21] -Input [2]: [c_customer_sk#21, ssales#27] +(37) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] -(40) Sort [codegen id : 3] +(38) CometSort Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21 ASC NULLS FIRST], false, 0 +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] -(41) SortMergeJoin [codegen id : 5] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#21] -Join type: LeftSemi -Join condition: None +(39) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi -(42) Project [codegen id : 5] -Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +(40) CometProject Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] + +(41) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#27, d_year#28, d_moy#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) + +(43) CometProject +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Arguments: [d_date_sk#27], [d_date_sk#27] + +(44) CometBroadcastExchange +Input [1]: [d_date_sk#27] +Arguments: [d_date_sk#27] + +(45) CometBroadcastHashJoin +Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [d_date_sk#27] +Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight + +(46) CometProject +Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] +Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] + +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] +ReadSchema: struct + +(48) ReusedExchange [Reuses operator id: 20] +Output [1]: [item_sk#36] + +(49) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [item_sk#36] +Arguments: [ws_item_sk#31], [item_sk#36], LeftSemi, BuildRight + +(50) CometProject +Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] + +(51) CometExchange +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(52) CometSort +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 34] +Output [3]: [c_customer_sk#37, sum#38, isEmpty#39] + +(54) CometHashAggregate +Input [3]: [c_customer_sk#37, sum#38, isEmpty#39] +Keys [1]: [c_customer_sk#37] +Functions [1]: [sum((cast(ss_quantity#40 as decimal(10,0)) * ss_sales_price#41))] + +(55) CometFilter +Input [2]: [c_customer_sk#37, ssales#42] +Condition : (isnotnull(ssales#42) AND (cast(ssales#42 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(56) CometProject +Input [2]: [c_customer_sk#37, ssales#42] +Arguments: [c_customer_sk#37], [c_customer_sk#37] + +(57) CometSort +Input [1]: [c_customer_sk#37] +Arguments: [c_customer_sk#37], [c_customer_sk#37 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [c_customer_sk#37] +Arguments: [ws_bill_customer_sk#32], [c_customer_sk#37], LeftSemi + +(59) CometProject +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] + +(60) ReusedExchange [Reuses operator id: 44] +Output [1]: [d_date_sk#43] + +(61) CometBroadcastHashJoin +Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [d_date_sk#43] +Arguments: [ws_sold_date_sk#35], [d_date_sk#43], Inner, BuildRight + +(62) CometProject +Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#43] +Arguments: [sales#44], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#44] + +(63) CometUnion +Child 0 Input [1]: [sales#30] +Child 1 Input [1]: [sales#44] + +(64) CometHashAggregate +Input [1]: [sales#30] +Keys: [] +Functions [1]: [partial_sum(sales#30)] -(43) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30, d_year#31, d_moy#32] - -(44) CometFilter -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Condition : ((((isnotnull(d_year#31) AND isnotnull(d_moy#32)) AND (d_year#31 = 2000)) AND (d_moy#32 = 2)) AND isnotnull(d_date_sk#30)) - -(45) CometProject -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30], [d_date_sk#30] - -(46) ColumnarToRow [codegen id : 4] -Input [1]: [d_date_sk#30] - -(47) BroadcastExchange -Input [1]: [d_date_sk#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(48) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#30] -Join type: Inner -Join condition: None - -(49) Project [codegen id : 5] -Output [1]: [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#33] -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#30] - -(50) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] - -(51) ReusedExchange [Reuses operator id: 20] -Output [1]: [item_sk#39] - -(52) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Right output [1]: [item_sk#39] -Arguments: [ws_item_sk#34], [item_sk#39], LeftSemi, BuildRight - -(53) CometProject -Input [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38], [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] - -(54) CometExchange -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: hashpartitioning(ws_bill_customer_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(55) CometSort -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38], [ws_bill_customer_sk#35 ASC NULLS FIRST] - -(56) ColumnarToRow [codegen id : 6] -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] - -(57) ReusedExchange [Reuses operator id: 36] -Output [3]: [c_customer_sk#40, sum#41, isEmpty#42] +(65) CometExchange +Input [2]: [sum#45, isEmpty#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(58) HashAggregate [codegen id : 8] -Input [3]: [c_customer_sk#40, sum#41, isEmpty#42] -Keys [1]: [c_customer_sk#40] -Functions [1]: [sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))#26] -Results [2]: [c_customer_sk#40, sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))#26 AS ssales#45] +(66) CometHashAggregate +Input [2]: [sum#45, isEmpty#46] +Keys: [] +Functions [1]: [sum(sales#30)] -(59) Filter [codegen id : 8] -Input [2]: [c_customer_sk#40, ssales#45] -Condition : (isnotnull(ssales#45) AND (cast(ssales#45 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#28, [id=#29]))) +(67) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(sales)#47] -(60) Project [codegen id : 8] -Output [1]: [c_customer_sk#40] -Input [2]: [c_customer_sk#40, ssales#45] +===== Subqueries ===== -(61) Sort [codegen id : 8] -Input [1]: [c_customer_sk#40] -Arguments: [c_customer_sk#40 ASC NULLS FIRST], false, 0 +Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (85) ++- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometHashAggregate (81) + +- CometExchange (80) + +- CometHashAggregate (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometFilter (69) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) + + +(68) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#51)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(69) CometFilter +Input [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Condition : isnotnull(ss_customer_sk#48) + +(70) ReusedExchange [Reuses operator id: 30] +Output [1]: [c_customer_sk#52] + +(71) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Right output [1]: [c_customer_sk#52] +Arguments: [ss_customer_sk#48], [c_customer_sk#52], Inner, BuildRight + +(72) CometProject +Input [5]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Arguments: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] + +(73) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#53, d_year#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(74) CometFilter +Input [2]: [d_date_sk#53, d_year#54] +Condition : (d_year#54 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#53)) -(62) SortMergeJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#35] -Right keys [1]: [c_customer_sk#40] -Join type: LeftSemi -Join condition: None +(75) CometProject +Input [2]: [d_date_sk#53, d_year#54] +Arguments: [d_date_sk#53], [d_date_sk#53] -(63) Project [codegen id : 10] -Output [3]: [ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] +(76) CometBroadcastExchange +Input [1]: [d_date_sk#53] +Arguments: [d_date_sk#53] -(64) ReusedExchange [Reuses operator id: 47] -Output [1]: [d_date_sk#46] +(77) CometBroadcastHashJoin +Left output [4]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Right output [1]: [d_date_sk#53] +Arguments: [ss_sold_date_sk#51], [d_date_sk#53], Inner, BuildRight -(65) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#38] -Right keys [1]: [d_date_sk#46] -Join type: Inner -Join condition: None +(78) CometProject +Input [5]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52, d_date_sk#53] +Arguments: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] -(66) Project [codegen id : 10] -Output [1]: [(cast(ws_quantity#36 as decimal(10,0)) * ws_list_price#37) AS sales#47] -Input [4]: [ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38, d_date_sk#46] - -(67) Union +(79) CometHashAggregate +Input [3]: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] +Keys [1]: [c_customer_sk#52] +Functions [1]: [partial_sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] -(68) HashAggregate [codegen id : 11] -Input [1]: [sales#33] -Keys: [] -Functions [1]: [partial_sum(sales#33)] -Aggregate Attributes [2]: [sum#48, isEmpty#49] -Results [2]: [sum#50, isEmpty#51] +(80) CometExchange +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Arguments: hashpartitioning(c_customer_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(69) Exchange -Input [2]: [sum#50, isEmpty#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(81) CometHashAggregate +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Keys [1]: [c_customer_sk#52] +Functions [1]: [sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] -(70) HashAggregate [codegen id : 12] -Input [2]: [sum#50, isEmpty#51] +(82) CometHashAggregate +Input [1]: [csales#57] Keys: [] -Functions [1]: [sum(sales#33)] -Aggregate Attributes [1]: [sum(sales#33)#52] -Results [1]: [sum(sales#33)#52 AS sum(sales)#53] +Functions [1]: [partial_max(csales#57)] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 38 Hosting Expression = Subquery scalar-subquery#28, [id=#29] -* HashAggregate (88) -+- Exchange (87) - +- * HashAggregate (86) - +- * HashAggregate (85) - +- Exchange (84) - +- * HashAggregate (83) - +- * ColumnarToRow (82) - +- CometProject (81) - +- CometBroadcastHashJoin (80) - :- CometProject (75) - : +- CometBroadcastHashJoin (74) - : :- CometFilter (72) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (71) - : +- ReusedExchange (73) - +- CometBroadcastExchange (79) - +- CometProject (78) - +- CometFilter (77) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (76) - - -(71) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] -Arguments: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] - -(72) CometFilter -Input [4]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] -Condition : isnotnull(ss_customer_sk#54) - -(73) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#58] - -(74) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] -Right output [1]: [c_customer_sk#58] -Arguments: [ss_customer_sk#54], [c_customer_sk#58], Inner, BuildRight - -(75) CometProject -Input [5]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58] -Arguments: [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58], [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58] - -(76) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#59, d_year#60] -Arguments: [d_date_sk#59, d_year#60] +(83) CometExchange +Input [1]: [max#58] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(77) CometFilter -Input [2]: [d_date_sk#59, d_year#60] -Condition : (d_year#60 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#59)) - -(78) CometProject -Input [2]: [d_date_sk#59, d_year#60] -Arguments: [d_date_sk#59], [d_date_sk#59] - -(79) CometBroadcastExchange -Input [1]: [d_date_sk#59] -Arguments: [d_date_sk#59] - -(80) CometBroadcastHashJoin -Left output [4]: [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58] -Right output [1]: [d_date_sk#59] -Arguments: [ss_sold_date_sk#57], [d_date_sk#59], Inner, BuildRight - -(81) CometProject -Input [5]: [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58, d_date_sk#59] -Arguments: [ss_quantity#55, ss_sales_price#56, c_customer_sk#58], [ss_quantity#55, ss_sales_price#56, c_customer_sk#58] - -(82) ColumnarToRow [codegen id : 1] -Input [3]: [ss_quantity#55, ss_sales_price#56, c_customer_sk#58] - -(83) HashAggregate [codegen id : 1] -Input [3]: [ss_quantity#55, ss_sales_price#56, c_customer_sk#58] -Keys [1]: [c_customer_sk#58] -Functions [1]: [partial_sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))] -Aggregate Attributes [2]: [sum#61, isEmpty#62] -Results [3]: [c_customer_sk#58, sum#63, isEmpty#64] - -(84) Exchange -Input [3]: [c_customer_sk#58, sum#63, isEmpty#64] -Arguments: hashpartitioning(c_customer_sk#58, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(85) HashAggregate [codegen id : 2] -Input [3]: [c_customer_sk#58, sum#63, isEmpty#64] -Keys [1]: [c_customer_sk#58] -Functions [1]: [sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))#65] -Results [1]: [sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))#65 AS csales#66] - -(86) HashAggregate [codegen id : 2] -Input [1]: [csales#66] +(84) CometHashAggregate +Input [1]: [max#58] Keys: [] -Functions [1]: [partial_max(csales#66)] -Aggregate Attributes [1]: [max#67] -Results [1]: [max#68] +Functions [1]: [max(csales#57)] -(87) Exchange -Input [1]: [max#68] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(88) HashAggregate [codegen id : 3] -Input [1]: [max#68] -Keys: [] -Functions [1]: [max(csales#66)] -Aggregate Attributes [1]: [max(csales#66)#69] -Results [1]: [max(csales#66)#69 AS tpcds_cmax#70] +(85) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#59] -Subquery:2 Hosting operator id = 59 Hosting Expression = ReusedSubquery Subquery scalar-subquery#28, [id=#29] +Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/extended.txt new file mode 100644 index 0000000000..a914b98c8e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/extended.txt @@ -0,0 +1,121 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometSort + : +- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 115 out of 117 eligible operators (98%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/simplified.txt index aa333d7ff5..3604457d42 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/simplified.txt @@ -1,120 +1,91 @@ -WholeStageCodegen (12) - HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (5) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [item_sk] #3 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #4 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (3) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (3) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (2) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (1) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - ReusedExchange [c_customer_sk] #8 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (2) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometHashAggregate [sum,isEmpty] [sum(sales),sum(sales)] + CometExchange #1 + CometHashAggregate [sales] [sum,isEmpty] + CometUnion [sales] + CometProject [cs_quantity,cs_list_price] [sales] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #8 - CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (10) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #13 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (8) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [d_date_sk] #12 + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [ws_quantity,ws_list_price] [sales] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #13 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/explain.txt index f02a51d2f8..df65b9cb4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/explain.txt @@ -1,116 +1,124 @@ == Physical Plan == -TakeOrderedAndProject (91) -+- Union (90) - :- * HashAggregate (66) - : +- Exchange (65) - : +- * HashAggregate (64) - : +- * Project (63) - : +- * BroadcastHashJoin Inner BuildRight (62) - : :- * Project (56) - : : +- * BroadcastHashJoin Inner BuildRight (55) - : : :- * SortMergeJoin LeftSemi (42) - : : : :- * ColumnarToRow (26) - : : : : +- CometSort (25) - : : : : +- CometExchange (24) - : : : : +- CometProject (23) - : : : : +- CometBroadcastHashJoin (22) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometBroadcastExchange (21) - : : : : +- CometProject (20) - : : : : +- CometFilter (19) - : : : : +- CometHashAggregate (18) - : : : : +- CometExchange (17) - : : : : +- CometHashAggregate (16) - : : : : +- CometProject (15) - : : : : +- CometBroadcastHashJoin (14) - : : : : :- CometProject (10) - : : : : : +- CometBroadcastHashJoin (9) - : : : : : :- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : : +- CometBroadcastExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (5) - : : : : +- CometBroadcastExchange (13) - : : : : +- CometFilter (12) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : +- * Sort (41) - : : : +- * Project (40) - : : : +- * Filter (39) - : : : +- * HashAggregate (38) - : : : +- Exchange (37) - : : : +- * HashAggregate (36) - : : : +- * ColumnarToRow (35) - : : : +- CometProject (34) - : : : +- CometBroadcastHashJoin (33) - : : : :- CometProject (29) - : : : : +- CometFilter (28) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (27) - : : : +- CometBroadcastExchange (32) - : : : +- CometFilter (31) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (30) - : : +- BroadcastExchange (54) - : : +- * SortMergeJoin LeftSemi (53) - : : :- * ColumnarToRow (47) - : : : +- CometSort (46) - : : : +- CometExchange (45) - : : : +- CometFilter (44) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (43) - : : +- * Sort (52) - : : +- * Project (51) - : : +- * Filter (50) - : : +- * HashAggregate (49) - : : +- ReusedExchange (48) - : +- BroadcastExchange (61) - : +- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometFilter (58) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (57) - +- * HashAggregate (89) - +- Exchange (88) - +- * HashAggregate (87) - +- * Project (86) - +- * BroadcastHashJoin Inner BuildRight (85) - :- * Project (83) - : +- * BroadcastHashJoin Inner BuildRight (82) - : :- * SortMergeJoin LeftSemi (80) - : : :- * ColumnarToRow (74) - : : : +- CometSort (73) - : : : +- CometExchange (72) - : : : +- CometProject (71) - : : : +- CometBroadcastHashJoin (70) - : : : :- CometFilter (68) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (67) - : : : +- ReusedExchange (69) - : : +- * Sort (79) - : : +- * Project (78) - : : +- * Filter (77) - : : +- * HashAggregate (76) - : : +- ReusedExchange (75) - : +- ReusedExchange (81) - +- ReusedExchange (84) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +* CometColumnarToRow (88) ++- CometTakeOrderedAndProject (87) + +- CometUnion (86) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometSortMergeJoin (40) + : : : :- CometSort (25) + : : : : +- CometExchange (24) + : : : : +- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometHashAggregate (18) + : : : : +- CometExchange (17) + : : : : +- CometHashAggregate (16) + : : : : +- CometProject (15) + : : : : +- CometBroadcastHashJoin (14) + : : : : :- CometProject (10) + : : : : : +- CometBroadcastHashJoin (9) + : : : : : :- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : : +- CometBroadcastExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (5) + : : : : +- CometBroadcastExchange (13) + : : : : +- CometFilter (12) + : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : +- CometSort (39) + : : : +- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometHashAggregate (36) + : : : +- CometExchange (35) + : : : +- CometHashAggregate (34) + : : : +- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) + : : : +- CometBroadcastExchange (31) + : : : +- CometFilter (30) + : : : +- CometNativeScan parquet spark_catalog.default.customer (29) + : : +- CometBroadcastExchange (52) + : : +- CometProject (51) + : : +- CometSortMergeJoin (50) + : : :- CometSort (44) + : : : +- CometExchange (43) + : : : +- CometFilter (42) + : : : +- CometNativeScan parquet spark_catalog.default.customer (41) + : : +- CometSort (49) + : : +- CometProject (48) + : : +- CometFilter (47) + : : +- CometHashAggregate (46) + : : +- ReusedExchange (45) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + +- CometHashAggregate (85) + +- CometExchange (84) + +- CometHashAggregate (83) + +- CometProject (82) + +- CometBroadcastHashJoin (81) + :- CometProject (79) + : +- CometBroadcastHashJoin (78) + : :- CometSortMergeJoin (76) + : : :- CometSort (70) + : : : +- CometExchange (69) + : : : +- CometProject (68) + : : : +- CometBroadcastHashJoin (67) + : : : :- CometFilter (65) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (64) + : : : +- ReusedExchange (66) + : : +- CometSort (75) + : : +- CometProject (74) + : : +- CometFilter (73) + : : +- CometHashAggregate (72) + : : +- ReusedExchange (71) + : +- ReusedExchange (77) + +- ReusedExchange (80) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(5) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct (6) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] @@ -133,9 +141,12 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(11) CometNativeScan: `spark_catalog`.`default`.`item` +(11) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (12) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] @@ -197,407 +208,393 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(26) ColumnarToRow [codegen id : 1] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(27) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct -(28) CometFilter +(27) CometFilter Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Condition : isnotnull(ss_customer_sk#17) -(29) CometProject +(28) CometProject Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(30) CometNativeScan: `spark_catalog`.`default`.`customer` +(29) CometNativeScan parquet spark_catalog.default.customer Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(31) CometFilter +(30) CometFilter Input [1]: [c_customer_sk#21] Condition : isnotnull(c_customer_sk#21) -(32) CometBroadcastExchange +(31) CometBroadcastExchange Input [1]: [c_customer_sk#21] Arguments: [c_customer_sk#21] -(33) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] Right output [1]: [c_customer_sk#21] Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight -(34) CometProject +(33) CometProject Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -(35) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(36) HashAggregate [codegen id : 2] +(34) CometHashAggregate Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] Keys [1]: [c_customer_sk#21] Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [2]: [sum#22, isEmpty#23] -Results [3]: [c_customer_sk#21, sum#24, isEmpty#25] -(37) Exchange -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(35) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(38) HashAggregate [codegen id : 3] -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] +(36) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] Keys [1]: [c_customer_sk#21] Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26] -Results [2]: [c_customer_sk#21, sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26 AS ssales#27] -(39) Filter [codegen id : 3] -Input [2]: [c_customer_sk#21, ssales#27] -Condition : (isnotnull(ssales#27) AND (cast(ssales#27 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#28, [id=#29]))) +(37) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) -(40) Project [codegen id : 3] -Output [1]: [c_customer_sk#21] -Input [2]: [c_customer_sk#21, ssales#27] +(38) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] -(41) Sort [codegen id : 3] +(39) CometSort Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21 ASC NULLS FIRST], false, 0 - -(42) SortMergeJoin [codegen id : 9] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#21] -Join type: LeftSemi -Join condition: None +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] -(43) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: [c_customer_sk#30, c_first_name#31, c_last_name#32] +(40) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi -(44) CometFilter -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Condition : isnotnull(c_customer_sk#30) +(41) CometNativeScan parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(45) CometExchange -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: hashpartitioning(c_customer_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(42) CometFilter +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Condition : isnotnull(c_customer_sk#27) -(46) CometSort -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: [c_customer_sk#30, c_first_name#31, c_last_name#32], [c_customer_sk#30 ASC NULLS FIRST] +(43) CometExchange +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(47) ColumnarToRow [codegen id : 4] -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] +(44) CometSort +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] -(48) ReusedExchange [Reuses operator id: 37] -Output [3]: [c_customer_sk#21, sum#24, isEmpty#25] +(45) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] -(49) HashAggregate [codegen id : 6] -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] +(46) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] Keys [1]: [c_customer_sk#21] Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26] -Results [2]: [c_customer_sk#21, sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26 AS ssales#27] -(50) Filter [codegen id : 6] -Input [2]: [c_customer_sk#21, ssales#27] -Condition : (isnotnull(ssales#27) AND (cast(ssales#27 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#28, [id=#29]))) +(47) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) -(51) Project [codegen id : 6] -Output [1]: [c_customer_sk#21] -Input [2]: [c_customer_sk#21, ssales#27] +(48) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] -(52) Sort [codegen id : 6] +(49) CometSort Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21 ASC NULLS FIRST], false, 0 - -(53) SortMergeJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [c_customer_sk#21] -Join type: LeftSemi -Join condition: None - -(54) BroadcastExchange -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(55) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#30] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 9] -Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#31, c_last_name#32] -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#30, c_first_name#31, c_last_name#32] - -(57) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_year#34, d_moy#35] - -(58) CometFilter -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2000)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(59) CometProject -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(60) ColumnarToRow [codegen id : 8] -Input [1]: [d_date_sk#33] - -(61) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(62) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 9] -Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#31, c_last_name#32] -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#31, c_last_name#32, d_date_sk#33] - -(64) HashAggregate [codegen id : 9] -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#31, c_last_name#32] -Keys [2]: [c_last_name#32, c_first_name#31] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(50) CometSortMergeJoin +Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi + +(51) CometProject +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#31] + +(52) CometBroadcastExchange +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] + +(53) CometBroadcastHashJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight + +(54) CometProject +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] + +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) + +(57) CometProject +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] + +(59) CometBroadcastHashJoin +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight + +(60) CometProject +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] + +(61) CometHashAggregate +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [2]: [sum#36, isEmpty#37] -Results [4]: [c_last_name#32, c_first_name#31, sum#38, isEmpty#39] -(65) Exchange -Input [4]: [c_last_name#32, c_first_name#31, sum#38, isEmpty#39] -Arguments: hashpartitioning(c_last_name#32, c_first_name#31, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(62) CometExchange +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(66) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#32, c_first_name#31, sum#38, isEmpty#39] -Keys [2]: [c_last_name#32, c_first_name#31] +(63) CometHashAggregate +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#40] -Results [3]: [c_last_name#32, c_first_name#31, sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#40 AS sales#41] - -(67) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] - -(68) CometFilter -Input [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_bill_customer_sk#43) - -(69) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#47] - -(70) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Right output [1]: [item_sk#47] -Arguments: [ws_item_sk#42], [item_sk#47], LeftSemi, BuildRight - -(71) CometProject -Input [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46], [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] - -(72) CometExchange -Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: hashpartitioning(ws_bill_customer_sk#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(73) CometSort -Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46], [ws_bill_customer_sk#43 ASC NULLS FIRST] - -(74) ColumnarToRow [codegen id : 11] -Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] - -(75) ReusedExchange [Reuses operator id: 37] -Output [3]: [c_customer_sk#48, sum#49, isEmpty#50] - -(76) HashAggregate [codegen id : 13] -Input [3]: [c_customer_sk#48, sum#49, isEmpty#50] -Keys [1]: [c_customer_sk#48] -Functions [1]: [sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))#26] -Results [2]: [c_customer_sk#48, sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))#26 AS ssales#53] - -(77) Filter [codegen id : 13] -Input [2]: [c_customer_sk#48, ssales#53] -Condition : (isnotnull(ssales#53) AND (cast(ssales#53 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#28, [id=#29]))) - -(78) Project [codegen id : 13] -Output [1]: [c_customer_sk#48] -Input [2]: [c_customer_sk#48, ssales#53] - -(79) Sort [codegen id : 13] -Input [1]: [c_customer_sk#48] -Arguments: [c_customer_sk#48 ASC NULLS FIRST], false, 0 - -(80) SortMergeJoin [codegen id : 19] -Left keys [1]: [ws_bill_customer_sk#43] -Right keys [1]: [c_customer_sk#48] -Join type: LeftSemi -Join condition: None - -(81) ReusedExchange [Reuses operator id: 54] -Output [3]: [c_customer_sk#54, c_first_name#55, c_last_name#56] - -(82) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ws_bill_customer_sk#43] -Right keys [1]: [c_customer_sk#54] -Join type: Inner -Join condition: None - -(83) Project [codegen id : 19] -Output [5]: [ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_first_name#55, c_last_name#56] -Input [7]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_customer_sk#54, c_first_name#55, c_last_name#56] - -(84) ReusedExchange [Reuses operator id: 61] -Output [1]: [d_date_sk#57] - -(85) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ws_sold_date_sk#46] -Right keys [1]: [d_date_sk#57] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 19] -Output [4]: [ws_quantity#44, ws_list_price#45, c_first_name#55, c_last_name#56] -Input [6]: [ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_first_name#55, c_last_name#56, d_date_sk#57] - -(87) HashAggregate [codegen id : 19] -Input [4]: [ws_quantity#44, ws_list_price#45, c_first_name#55, c_last_name#56] -Keys [2]: [c_last_name#56, c_first_name#55] -Functions [1]: [partial_sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))] -Aggregate Attributes [2]: [sum#58, isEmpty#59] -Results [4]: [c_last_name#56, c_first_name#55, sum#60, isEmpty#61] - -(88) Exchange -Input [4]: [c_last_name#56, c_first_name#55, sum#60, isEmpty#61] -Arguments: hashpartitioning(c_last_name#56, c_first_name#55, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(89) HashAggregate [codegen id : 20] -Input [4]: [c_last_name#56, c_first_name#55, sum#60, isEmpty#61] -Keys [2]: [c_last_name#56, c_first_name#55] -Functions [1]: [sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))] -Aggregate Attributes [1]: [sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))#62] -Results [3]: [c_last_name#56, c_first_name#55, sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))#62 AS sales#63] - -(90) Union - -(91) TakeOrderedAndProject -Input [3]: [c_last_name#32, c_first_name#31, sales#41] -Arguments: 100, [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, sales#41 ASC NULLS FIRST], [c_last_name#32, c_first_name#31, sales#41] + +(64) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(65) CometFilter +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) + +(66) ReusedExchange [Reuses operator id: 21] +Output [1]: [item_sk#42] + +(67) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#42] +Arguments: [ws_item_sk#37], [item_sk#42], LeftSemi, BuildRight + +(68) CometProject +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] + +(69) CometExchange +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(70) CometSort +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] + +(71) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#43, sum#44, isEmpty#45] + +(72) CometHashAggregate +Input [3]: [c_customer_sk#43, sum#44, isEmpty#45] +Keys [1]: [c_customer_sk#43] +Functions [1]: [sum((cast(ss_quantity#46 as decimal(10,0)) * ss_sales_price#47))] + +(73) CometFilter +Input [2]: [c_customer_sk#43, ssales#48] +Condition : (isnotnull(ssales#48) AND (cast(ssales#48 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(74) CometProject +Input [2]: [c_customer_sk#43, ssales#48] +Arguments: [c_customer_sk#43], [c_customer_sk#43] + +(75) CometSort +Input [1]: [c_customer_sk#43] +Arguments: [c_customer_sk#43], [c_customer_sk#43 ASC NULLS FIRST] + +(76) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#43] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#43], LeftSemi + +(77) ReusedExchange [Reuses operator id: 52] +Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] + +(78) CometBroadcastHashJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#49], Inner, BuildRight + +(79) CometProject +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] + +(80) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#52] + +(81) CometBroadcastHashJoin +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] +Right output [1]: [d_date_sk#52] +Arguments: [ws_sold_date_sk#41], [d_date_sk#52], Inner, BuildRight + +(82) CometProject +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51, d_date_sk#52] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] + +(83) CometHashAggregate +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(84) CometExchange +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(85) CometHashAggregate +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(86) CometUnion +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Child 1 Input [3]: [c_last_name#51, c_first_name#50, sales#56] + +(87) CometTakeOrderedAndProject +Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#55 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#55]), [c_last_name#31, c_first_name#30, sales#55], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#55 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#55] + +(88) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] ===== Subqueries ===== -Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquery#28, [id=#29] -* HashAggregate (109) -+- Exchange (108) - +- * HashAggregate (107) - +- * HashAggregate (106) - +- Exchange (105) - +- * HashAggregate (104) - +- * ColumnarToRow (103) - +- CometProject (102) - +- CometBroadcastHashJoin (101) - :- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometFilter (93) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (92) - : +- ReusedExchange (94) - +- CometBroadcastExchange (100) - +- CometProject (99) - +- CometFilter (98) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (97) - - -(92) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] -Arguments: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] - -(93) CometFilter -Input [4]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] -Condition : isnotnull(ss_customer_sk#64) - -(94) ReusedExchange [Reuses operator id: 32] -Output [1]: [c_customer_sk#68] - -(95) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] -Right output [1]: [c_customer_sk#68] -Arguments: [ss_customer_sk#64], [c_customer_sk#68], Inner, BuildRight +Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometNativeScan parquet spark_catalog.default.date_dim (94) + + +(89) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#60)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(90) CometFilter +Input [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Condition : isnotnull(ss_customer_sk#57) + +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#61] + +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Right output [1]: [c_customer_sk#61] +Arguments: [ss_customer_sk#57], [c_customer_sk#61], Inner, BuildRight + +(93) CometProject +Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] + +(94) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#62, d_year#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(95) CometFilter +Input [2]: [d_date_sk#62, d_year#63] +Condition : (d_year#63 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#62)) (96) CometProject -Input [5]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68] -Arguments: [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68], [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68] +Input [2]: [d_date_sk#62, d_year#63] +Arguments: [d_date_sk#62], [d_date_sk#62] -(97) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69, d_year#70] +(97) CometBroadcastExchange +Input [1]: [d_date_sk#62] +Arguments: [d_date_sk#62] -(98) CometFilter -Input [2]: [d_date_sk#69, d_year#70] -Condition : (d_year#70 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#69)) +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#60], [d_date_sk#62], Inner, BuildRight (99) CometProject -Input [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69], [d_date_sk#69] - -(100) CometBroadcastExchange -Input [1]: [d_date_sk#69] -Arguments: [d_date_sk#69] - -(101) CometBroadcastHashJoin -Left output [4]: [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68] -Right output [1]: [d_date_sk#69] -Arguments: [ss_sold_date_sk#67], [d_date_sk#69], Inner, BuildRight - -(102) CometProject -Input [5]: [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68, d_date_sk#69] -Arguments: [ss_quantity#65, ss_sales_price#66, c_customer_sk#68], [ss_quantity#65, ss_sales_price#66, c_customer_sk#68] - -(103) ColumnarToRow [codegen id : 1] -Input [3]: [ss_quantity#65, ss_sales_price#66, c_customer_sk#68] - -(104) HashAggregate [codegen id : 1] -Input [3]: [ss_quantity#65, ss_sales_price#66, c_customer_sk#68] -Keys [1]: [c_customer_sk#68] -Functions [1]: [partial_sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))] -Aggregate Attributes [2]: [sum#71, isEmpty#72] -Results [3]: [c_customer_sk#68, sum#73, isEmpty#74] - -(105) Exchange -Input [3]: [c_customer_sk#68, sum#73, isEmpty#74] -Arguments: hashpartitioning(c_customer_sk#68, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(106) HashAggregate [codegen id : 2] -Input [3]: [c_customer_sk#68, sum#73, isEmpty#74] -Keys [1]: [c_customer_sk#68] -Functions [1]: [sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))#75] -Results [1]: [sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))#75 AS csales#76] - -(107) HashAggregate [codegen id : 2] -Input [1]: [csales#76] +Input [5]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61, d_date_sk#62] +Arguments: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] + +(100) CometHashAggregate +Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] +Keys [1]: [c_customer_sk#61] +Functions [1]: [partial_sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(101) CometExchange +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Arguments: hashpartitioning(c_customer_sk#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(102) CometHashAggregate +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Keys [1]: [c_customer_sk#61] +Functions [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(103) CometHashAggregate +Input [1]: [csales#66] Keys: [] -Functions [1]: [partial_max(csales#76)] -Aggregate Attributes [1]: [max#77] -Results [1]: [max#78] +Functions [1]: [partial_max(csales#66)] -(108) Exchange -Input [1]: [max#78] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +(104) CometExchange +Input [1]: [max#67] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(109) HashAggregate [codegen id : 3] -Input [1]: [max#78] +(105) CometHashAggregate +Input [1]: [max#67] Keys: [] -Functions [1]: [max(csales#76)] -Aggregate Attributes [1]: [max(csales#76)#79] -Results [1]: [max(csales#76)#79 AS tpcds_cmax#80] +Functions [1]: [max(csales#66)] + +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#68] -Subquery:2 Hosting operator id = 50 Hosting Expression = ReusedSubquery Subquery scalar-subquery#28, [id=#29] +Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:3 Hosting operator id = 77 Hosting Expression = ReusedSubquery Subquery scalar-subquery#28, [id=#29] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/extended.txt new file mode 100644 index 0000000000..bdae9844c6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/extended.txt @@ -0,0 +1,173 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometSort + : : : +- CometProject + : : : +- CometFilter + : : : : +- Subquery + : : : : +- CometColumnarToRow + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometSort + : +- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 165 out of 169 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/simplified.txt index f29c0e5e18..2c389585a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/simplified.txt @@ -1,151 +1,113 @@ -TakeOrderedAndProject [c_last_name,c_first_name,sales] - Union - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #1 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_list_price,c_first_name,c_last_name] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [item_sk] #3 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #4 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (3) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (3) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (2) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (1) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - ReusedExchange [c_customer_sk] #8 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,sales] + CometUnion [c_last_name,c_first_name,sales] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] + CometExchange [c_last_name,c_first_name] #1 + CometHashAggregate [cs_quantity,cs_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (2) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #8 - CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (7) - SortMergeJoin [c_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #13 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - InputAdapter - WholeStageCodegen (6) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (20) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #15 - WholeStageCodegen (19) - HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_list_price,c_first_name,c_last_name] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #16 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (13) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] + CometSort [c_customer_sk,c_first_name,c_last_name] + CometExchange [c_customer_sk] #13 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 - InputAdapter - ReusedExchange [d_date_sk] #14 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] + CometExchange [c_last_name,c_first_name] #15 + CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #16 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/explain.txt index 5d63cafbe5..d6087012db 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] @@ -63,9 +71,12 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -92,291 +103,337 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometNativeScan: `spark_catalog`.`default`.`store` +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] +ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((isnotnull(i_color#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) = pale )) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#19, 20)) AS i_size#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) AS i_color#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#21, 10)) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) + +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#31, c_birth_country#29] + +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#32, 2)) AS ca_state#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)) AS ca_zip#36, ca_country#34] + +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] + +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometNativeScan parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] (55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight (56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] +(57) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight +(58) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) (59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [i_item_sk#60, i_current_price#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#62, 20)) AS i_size#66, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#63, 20)) AS i_color#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#64, 10)) AS i_units#68, i_manager_id#65] + +(60) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] + +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_item_sk#49], [i_item_sk#60], Inner, BuildRight -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Right output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_customer_sk#50], [c_customer_sk#69], Inner, BuildRight + +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72], [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#73, ca_zip#74, ca_country#75] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#72, s_zip#59] +Right keys [2]: [upper(ca_country#75), ca_zip#74] Join type: Inner Join condition: None -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72, ca_state#73, ca_zip#74, ca_country#75] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#76] +Results [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(71) CometColumnarExchange +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Arguments: hashpartitioning(c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#78] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#78] Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] +Functions [1]: [partial_avg(netpaid#78)] +Aggregate Attributes [2]: [sum#79, count#80] +Results [2]: [sum#81, count#82] + +(75) CometColumnarExchange +Input [2]: [sum#81, count#82] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#81, count#82] -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#81, count#82] Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] +Functions [1]: [avg(netpaid#78)] +Aggregate Attributes [1]: [avg(netpaid#78)#83] +Results [1]: [(0.05 * avg(netpaid#78)#83) AS (0.05 * avg(netpaid))#84] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/extended.txt new file mode 100644 index 0000000000..6b21db37a2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/extended.txt @@ -0,0 +1,96 @@ +Filter +: +- Subquery +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- Project +: +- BroadcastHashJoin +: :- CometColumnarToRow +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometSortMergeJoin +: : : : : :- CometSort +: : : : : : +- CometExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometSort +: : : : : +- CometExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.item +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.customer +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.customer_address ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/simplified.txt index c1bf41667f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/simplified.txt @@ -3,18 +3,56 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] @@ -25,62 +63,32 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/explain.txt index 829f96c1fb..f4e4f60060 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] @@ -63,9 +71,12 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -92,291 +103,337 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometNativeScan: `spark_catalog`.`default`.`store` +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] +ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((isnotnull(i_color#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) = chiffon )) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#19, 20)) AS i_size#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) AS i_color#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#21, 10)) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) + +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#31, c_birth_country#29] + +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#32, 2)) AS ca_state#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)) AS ca_zip#36, ca_country#34] + +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] + +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometNativeScan parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] (55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight (56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] +(57) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight +(58) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) (59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [i_item_sk#60, i_current_price#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#62, 20)) AS i_size#66, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#63, 20)) AS i_color#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#64, 10)) AS i_units#68, i_manager_id#65] + +(60) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] + +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_item_sk#49], [i_item_sk#60], Inner, BuildRight -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Right output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_customer_sk#50], [c_customer_sk#69], Inner, BuildRight + +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72], [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#73, ca_zip#74, ca_country#75] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#72, s_zip#59] +Right keys [2]: [upper(ca_country#75), ca_zip#74] Join type: Inner Join condition: None -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72, ca_state#73, ca_zip#74, ca_country#75] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#76] +Results [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(71) CometColumnarExchange +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Arguments: hashpartitioning(c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#78] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#78] Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] +Functions [1]: [partial_avg(netpaid#78)] +Aggregate Attributes [2]: [sum#79, count#80] +Results [2]: [sum#81, count#82] + +(75) CometColumnarExchange +Input [2]: [sum#81, count#82] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#81, count#82] -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#81, count#82] Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] +Functions [1]: [avg(netpaid#78)] +Aggregate Attributes [1]: [avg(netpaid#78)#83] +Results [1]: [(0.05 * avg(netpaid#78)#83) AS (0.05 * avg(netpaid))#84] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/extended.txt new file mode 100644 index 0000000000..6b21db37a2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/extended.txt @@ -0,0 +1,96 @@ +Filter +: +- Subquery +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- Project +: +- BroadcastHashJoin +: :- CometColumnarToRow +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometSortMergeJoin +: : : : : :- CometSort +: : : : : : +- CometExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometSort +: : : : : +- CometExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.item +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.customer +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.customer_address ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/simplified.txt index c1bf41667f..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/simplified.txt @@ -3,18 +3,56 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] @@ -25,62 +63,32 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/explain.txt index 9bd1ec11cd..ab182b5027 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) : :- CometProject (27) : : +- CometBroadcastHashJoin (26) : : :- CometProject (24) @@ -19,39 +19,51 @@ TakeOrderedAndProject (40) : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- ReusedExchange (33) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.item (34) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] @@ -70,9 +82,13 @@ Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] @@ -91,9 +107,12 @@ Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] @@ -116,9 +135,12 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] @@ -153,61 +175,82 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -(28) CometNativeScan: `spark_catalog`.`default`.`store` +(28) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (29) CometFilter Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Condition : isnotnull(s_store_sk#23) -(30) CometBroadcastExchange +(30) CometProject Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#26, s_store_name#25] + +(31) CometBroadcastExchange +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] -(31) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -Right output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight -(32) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] - -(33) ReusedExchange [Reuses operator id: 30] -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] - -(34) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Right output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_item_sk#1], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] - -(36) ColumnarToRow [codegen id : 1] -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] - -(37) HashAggregate [codegen id : 1] -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(33) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] + +(34) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(35) CometFilter +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) + +(36) CometProject +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#30, i_item_desc#29] + +(37) CometBroadcastExchange +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] + +(38) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight + +(39) CometProject +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] + +(40) CometHashAggregate +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum#29, sum#30, sum#31] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -(38) Exchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(41) CometExchange +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(42) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] -(39) HashAggregate [codegen id : 2] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(43) HashAggregate [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#35, sum(UnscaledValue(sr_net_loss#10))#36, sum(UnscaledValue(cs_net_profit#14))#37] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#35,17,2) AS store_sales_profit#38, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#36,17,2) AS store_returns_loss#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#37,17,2) AS catalog_sales_profit#40] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#34, sum(UnscaledValue(sr_net_loss#10))#35, sum(UnscaledValue(cs_net_profit#14))#36] +Results [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#34,17,2) AS store_sales_profit#37, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#35,17,2) AS store_returns_loss#38, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#36,17,2) AS catalog_sales_profit#39] -(40) TakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] -Arguments: 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] +(44) TakeOrderedAndProject +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] +Arguments: 100, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/extended.txt new file mode 100644 index 0000000000..f5b1edd374 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/extended.txt @@ -0,0 +1,49 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#1, i_item_desc#2, s_store_id#3, s_store_name#4, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#6,17,2) AS store_sales_profit#7, MakeDecimal(sum(UnscaledValue(sr_net_loss#8))#9,17,2) AS store_returns_loss#10, MakeDecimal(sum(UnscaledValue(cs_net_profit#11))#12,17,2) AS catalog_sales_profit#13)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 44 out of 46 eligible operators (95%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/simplified.txt index d51ec29fe4..4562cf03f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/simplified.txt @@ -1,44 +1,46 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_net_profit,sr_net_loss,cs_net_profit] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #6 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/explain.txt index 6edda09877..36aa35e54a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.promotion (21) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Arguments: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -64,9 +72,12 @@ Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#13, d_year#14] @@ -89,74 +100,80 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] +Right output [2]: [i_item_sk#15, i_item_id#17] Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [cs_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(26) ColumnarToRow [codegen id : 1] -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(27) HashAggregate [codegen id : 1] -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(20) CometProject +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] + +(21) CometNativeScan parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#19, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#20, 1)) = N)) AND isnotnull(p_promo_sk#18)) + +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] + +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] + +(25) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight + +(26) CometProject +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] + +(27) CometHashAggregate +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -(28) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(29) HashAggregate [codegen id : 2] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [4]: [avg(cs_quantity#4)#36, avg(UnscaledValue(cs_list_price#5))#37, avg(UnscaledValue(cs_coupon_amt#7))#38, avg(UnscaledValue(cs_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(cs_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(cs_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(cs_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(cs_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] -(30) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/extended.txt new file mode 100644 index 0000000000..7382341430 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/simplified.txt index 39633af4ee..55cf7624f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/simplified.txt @@ -1,34 +1,33 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (2) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] - CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] - CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #5 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] + CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] + CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/explain.txt index df9ba7f4b5..e8815d80e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/explain.txt @@ -1,51 +1,60 @@ == Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometExpand (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (32) ++- CometTakeOrderedAndProject (31) + +- CometHashAggregate (30) + +- CometExchange (29) + +- CometHashAggregate (28) + +- CometExpand (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- CometBroadcastExchange (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.item (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -64,9 +73,12 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#13, d_year#14] @@ -89,74 +101,84 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Condition : ((isnotnull(s_state#16) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) = TN)) AND isnotnull(s_store_sk#15)) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) AS s_state#17] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] +Right output [2]: [s_store_sk#15, s_state#17] Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] - -(25) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] - -(26) ColumnarToRow [codegen id : 1] -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] - -(27) HashAggregate [codegen id : 1] -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] + +(21) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) + +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] + +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] + +(27) CometExpand +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] + +(28) CometHashAggregate +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Results [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -(28) Exchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(29) CometExchange +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(29) HashAggregate [codegen id : 2] -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(30) CometHashAggregate +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#38, avg(UnscaledValue(ss_list_price#5))#39, avg(UnscaledValue(ss_coupon_amt#7))#40, avg(UnscaledValue(ss_sales_price#6))#41] -Results [7]: [i_item_id#19, s_state#20, cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint) AS g_state#42, avg(ss_quantity#4)#38 AS agg1#43, cast((avg(UnscaledValue(ss_list_price#5))#39 / 100.0) as decimal(11,6)) AS agg2#44, cast((avg(UnscaledValue(ss_coupon_amt#7))#40 / 100.0) as decimal(11,6)) AS agg3#45, cast((avg(UnscaledValue(ss_sales_price#6))#41 / 100.0) as decimal(11,6)) AS agg4#46] -(30) TakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] -Arguments: 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] +(31) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] + +(32) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/extended.txt new file mode 100644 index 0000000000..666d08fb88 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/extended.txt @@ -0,0 +1,34 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 31 out of 31 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/simplified.txt index e39b06308f..ff5133d342 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/simplified.txt @@ -1,34 +1,34 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - WholeStageCodegen (2) - HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state,spark_grouping_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id,s_state,spark_grouping_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/explain.txt index a917d8262b..985274408b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/explain.txt @@ -1,79 +1,88 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- Exchange (6) -: : : : : +- * HashAggregate (5) -: : : : : +- * ColumnarToRow (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- Exchange (16) -: : : : +- * HashAggregate (15) -: : : : +- * ColumnarToRow (14) -: : : : +- CometProject (13) -: : : : +- CometFilter (12) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- Exchange (28) -: : : +- * HashAggregate (27) -: : : +- * ColumnarToRow (26) -: : : +- CometProject (25) -: : : +- CometFilter (24) -: : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- Exchange (40) -: : +- * HashAggregate (39) -: : +- * ColumnarToRow (38) -: : +- CometProject (37) -: : +- CometFilter (36) -: : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- Exchange (52) -: +- * HashAggregate (51) -: +- * ColumnarToRow (50) -: +- CometProject (49) -: +- CometFilter (48) -: +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * ColumnarToRow (62) - +- CometProject (61) - +- CometFilter (60) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (59) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* BroadcastNestedLoopJoin Inner BuildRight (76) +:- * BroadcastNestedLoopJoin Inner BuildRight (63) +: :- * BroadcastNestedLoopJoin Inner BuildRight (50) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (37) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (24) +: : : : :- * CometColumnarToRow (11) +: : : : : +- CometHashAggregate (10) +: : : : : +- CometColumnarExchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * CometColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (23) +: : : : +- * CometColumnarToRow (22) +: : : : +- CometHashAggregate (21) +: : : : +- CometColumnarExchange (20) +: : : : +- * HashAggregate (19) +: : : : +- * HashAggregate (18) +: : : : +- * CometColumnarToRow (17) +: : : : +- CometExchange (16) +: : : : +- CometHashAggregate (15) +: : : : +- CometProject (14) +: : : : +- CometFilter (13) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (12) +: : : +- BroadcastExchange (36) +: : : +- * CometColumnarToRow (35) +: : : +- CometHashAggregate (34) +: : : +- CometColumnarExchange (33) +: : : +- * HashAggregate (32) +: : : +- * HashAggregate (31) +: : : +- * CometColumnarToRow (30) +: : : +- CometExchange (29) +: : : +- CometHashAggregate (28) +: : : +- CometProject (27) +: : : +- CometFilter (26) +: : : +- CometNativeScan parquet spark_catalog.default.store_sales (25) +: : +- BroadcastExchange (49) +: : +- * CometColumnarToRow (48) +: : +- CometHashAggregate (47) +: : +- CometColumnarExchange (46) +: : +- * HashAggregate (45) +: : +- * HashAggregate (44) +: : +- * CometColumnarToRow (43) +: : +- CometExchange (42) +: : +- CometHashAggregate (41) +: : +- CometProject (40) +: : +- CometFilter (39) +: : +- CometNativeScan parquet spark_catalog.default.store_sales (38) +: +- BroadcastExchange (62) +: +- * CometColumnarToRow (61) +: +- CometHashAggregate (60) +: +- CometColumnarExchange (59) +: +- * HashAggregate (58) +: +- * HashAggregate (57) +: +- * CometColumnarToRow (56) +: +- CometExchange (55) +: +- CometHashAggregate (54) +: +- CometProject (53) +: +- CometFilter (52) +: +- CometNativeScan parquet spark_catalog.default.store_sales (51) ++- BroadcastExchange (75) + +- * CometColumnarToRow (74) + +- CometHashAggregate (73) + +- CometColumnarExchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- * CometColumnarToRow (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometProject (66) + +- CometFilter (65) + +- CometNativeScan parquet spark_catalog.default.store_sales (64) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Arguments: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))] +ReadSchema: struct (2) CometFilter Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] @@ -83,337 +92,346 @@ Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quanti Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] Arguments: [ss_list_price#3], [ss_list_price#3] -(4) ColumnarToRow [codegen id : 1] -Input [1]: [ss_list_price#3] - -(5) HashAggregate [codegen id : 1] +(4) CometHashAggregate Input [1]: [ss_list_price#3] Keys [1]: [ss_list_price#3] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] -(6) Exchange -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(5) CometExchange +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(7) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] +(6) CometColumnarToRow [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] + +(7) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Keys [1]: [ss_list_price#3] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10] +Results [4]: [ss_list_price#3, sum#6, count#7, count#8] -(8) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] +(8) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [4]: [sum#8, count#9, count#10, count#12] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] +Results [4]: [sum#6, count#7, count#8, count#12] -(9) Exchange -Input [4]: [sum#8, count#9, count#10, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(9) CometColumnarExchange +Input [4]: [sum#6, count#7, count#8, count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(10) HashAggregate [codegen id : 18] -Input [4]: [sum#8, count#9, count#10, count#12] +(10) CometHashAggregate +Input [4]: [sum#6, count#7, count#8, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#7 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(11) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(11) CometColumnarToRow [codegen id : 12] +Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] + +(12) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] +ReadSchema: struct -(12) CometFilter +(13) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(13) CometProject +(14) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Arguments: [ss_list_price#18], [ss_list_price#18] -(14) ColumnarToRow [codegen id : 3] -Input [1]: [ss_list_price#18] - -(15) HashAggregate [codegen id : 3] +(15) CometHashAggregate Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] -(16) Exchange -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(17) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] +(17) CometColumnarToRow [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] + +(18) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] +Results [4]: [ss_list_price#18, sum#21, count#22, count#23] -(18) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] +(19) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [4]: [sum#23, count#24, count#25, count#27] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] +Results [4]: [sum#21, count#22, count#23, count#27] -(19) Exchange -Input [4]: [sum#23, count#24, count#25, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(20) CometColumnarExchange +Input [4]: [sum#21, count#22, count#23, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(20) HashAggregate [codegen id : 5] -Input [4]: [sum#23, count#24, count#25, count#27] +(21) CometHashAggregate +Input [4]: [sum#21, count#22, count#23, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#21 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#22 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(21) BroadcastExchange +(22) CometColumnarToRow [codegen id : 3] +Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] + +(23) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(22) BroadcastNestedLoopJoin [codegen id : 18] +(24) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(23) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(25) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Arguments: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] +ReadSchema: struct -(24) CometFilter +(26) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(25) CometProject +(27) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Arguments: [ss_list_price#33], [ss_list_price#33] -(26) ColumnarToRow [codegen id : 6] -Input [1]: [ss_list_price#33] - -(27) HashAggregate [codegen id : 6] +(28) CometHashAggregate Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] -(28) Exchange -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(29) CometExchange +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(30) CometColumnarToRow [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -(29) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] +(31) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] +Results [4]: [ss_list_price#33, sum#36, count#37, count#38] -(30) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] +(32) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [4]: [sum#38, count#39, count#40, count#42] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] +Results [4]: [sum#36, count#37, count#38, count#42] -(31) Exchange -Input [4]: [sum#38, count#39, count#40, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(33) CometColumnarExchange +Input [4]: [sum#36, count#37, count#38, count#42] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(32) HashAggregate [codegen id : 8] -Input [4]: [sum#38, count#39, count#40, count#42] +(34) CometHashAggregate +Input [4]: [sum#36, count#37, count#38, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#36 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#37 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(33) BroadcastExchange +(35) CometColumnarToRow [codegen id : 5] +Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] + +(36) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(34) BroadcastNestedLoopJoin [codegen id : 18] +(37) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(35) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(38) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Arguments: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] +ReadSchema: struct -(36) CometFilter +(39) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(37) CometProject +(40) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Arguments: [ss_list_price#48], [ss_list_price#48] -(38) ColumnarToRow [codegen id : 9] -Input [1]: [ss_list_price#48] - -(39) HashAggregate [codegen id : 9] +(41) CometHashAggregate Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] -(40) Exchange -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(42) CometExchange +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(43) CometColumnarToRow [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -(41) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] +(44) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] +Results [4]: [ss_list_price#48, sum#51, count#52, count#53] -(42) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] +(45) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [4]: [sum#53, count#54, count#55, count#57] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] +Results [4]: [sum#51, count#52, count#53, count#57] -(43) Exchange -Input [4]: [sum#53, count#54, count#55, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +(46) CometColumnarExchange +Input [4]: [sum#51, count#52, count#53, count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(44) HashAggregate [codegen id : 11] -Input [4]: [sum#53, count#54, count#55, count#57] +(47) CometHashAggregate +Input [4]: [sum#51, count#52, count#53, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#51 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#52 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(45) BroadcastExchange +(48) CometColumnarToRow [codegen id : 7] +Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] + +(49) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(46) BroadcastNestedLoopJoin [codegen id : 18] +(50) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(51) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Arguments: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] +ReadSchema: struct -(48) CometFilter +(52) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(49) CometProject +(53) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Arguments: [ss_list_price#63], [ss_list_price#63] -(50) ColumnarToRow [codegen id : 12] -Input [1]: [ss_list_price#63] - -(51) HashAggregate [codegen id : 12] +(54) CometHashAggregate Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] -(52) Exchange -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(55) CometExchange +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(56) CometColumnarToRow [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -(53) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] +(57) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] +Results [4]: [ss_list_price#63, sum#66, count#67, count#68] -(54) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] +(58) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [4]: [sum#68, count#69, count#70, count#72] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] +Results [4]: [sum#66, count#67, count#68, count#72] -(55) Exchange -Input [4]: [sum#68, count#69, count#70, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +(59) CometColumnarExchange +Input [4]: [sum#66, count#67, count#68, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] -(56) HashAggregate [codegen id : 14] -Input [4]: [sum#68, count#69, count#70, count#72] +(60) CometHashAggregate +Input [4]: [sum#66, count#67, count#68, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#66 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#67 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(57) BroadcastExchange +(61) CometColumnarToRow [codegen id : 9] +Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] + +(62) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(58) BroadcastNestedLoopJoin [codegen id : 18] +(63) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(59) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(64) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Arguments: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] +ReadSchema: struct -(60) CometFilter +(65) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(61) CometProject +(66) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Arguments: [ss_list_price#78], [ss_list_price#78] -(62) ColumnarToRow [codegen id : 15] -Input [1]: [ss_list_price#78] - -(63) HashAggregate [codegen id : 15] +(67) CometHashAggregate Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] -(64) Exchange -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(68) CometExchange +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(65) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] +(69) CometColumnarToRow [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] + +(70) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] +Results [4]: [ss_list_price#78, sum#81, count#82, count#83] -(66) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] +(71) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [4]: [sum#83, count#84, count#85, count#87] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] +Results [4]: [sum#81, count#82, count#83, count#87] -(67) Exchange -Input [4]: [sum#83, count#84, count#85, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +(72) CometColumnarExchange +Input [4]: [sum#81, count#82, count#83, count#87] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] -(68) HashAggregate [codegen id : 17] -Input [4]: [sum#83, count#84, count#85, count#87] +(73) CometHashAggregate +Input [4]: [sum#81, count#82, count#83, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#81 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#82 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(69) BroadcastExchange +(74) CometColumnarToRow [codegen id : 11] +Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] + +(75) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(70) BroadcastNestedLoopJoin [codegen id : 18] +(76) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/extended.txt new file mode 100644 index 0000000000..3d8c2f298e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +BroadcastNestedLoopJoin +:- BroadcastNestedLoopJoin +: :- BroadcastNestedLoopJoin +: : :- BroadcastNestedLoopJoin +: : : :- BroadcastNestedLoopJoin +: : : : :- CometColumnarToRow +: : : : : +- CometHashAggregate +: : : : : +- CometColumnarExchange +: : : : : +- HashAggregate +: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : : +- CometColumnarToRow +: : : : : +- CometExchange +: : : : : +- CometHashAggregate +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : +- BroadcastExchange +: : : : +- CometColumnarToRow +: : : : +- CometHashAggregate +: : : : +- CometColumnarExchange +: : : : +- HashAggregate +: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : +- CometColumnarToRow +: : : : +- CometExchange +: : : : +- CometHashAggregate +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : +- BroadcastExchange +: : : +- CometColumnarToRow +: : : +- CometHashAggregate +: : : +- CometColumnarExchange +: : : +- HashAggregate +: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : +- CometColumnarToRow +: : : +- CometExchange +: : : +- CometHashAggregate +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : +- BroadcastExchange +: : +- CometColumnarToRow +: : +- CometHashAggregate +: : +- CometColumnarExchange +: : +- HashAggregate +: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : +- CometColumnarToRow +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometHashAggregate +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: +- CometColumnarToRow +: +- CometExchange +: +- CometHashAggregate +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.store_sales ++- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_sales + +Comet accelerated 42 out of 64 eligible operators (65%). Final plan contains 12 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/simplified.txt index 4a969c97db..a8540a4abb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/simplified.txt @@ -1,111 +1,105 @@ -WholeStageCodegen (18) +WholeStageCodegen (12) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #2 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B1_LP,B1_CNT,B1_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange #4 - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #5 - WholeStageCodegen (3) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B2_LP,B2_CNT,B2_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #4 + WholeStageCodegen (2) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #5 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #6 - WholeStageCodegen (8) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange #7 - WholeStageCodegen (7) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #8 - WholeStageCodegen (6) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B3_LP,B3_CNT,B3_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #7 + WholeStageCodegen (4) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #8 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #9 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] + WholeStageCodegen (7) + CometColumnarToRow InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #11 - WholeStageCodegen (9) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B4_LP,B4_CNT,B4_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #10 + WholeStageCodegen (6) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #11 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #12 - WholeStageCodegen (14) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] + WholeStageCodegen (9) + CometColumnarToRow InputAdapter - Exchange #13 - WholeStageCodegen (13) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #14 - WholeStageCodegen (12) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B5_LP,B5_CNT,B5_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #13 + WholeStageCodegen (8) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #14 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #15 - WholeStageCodegen (17) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - Exchange #16 - WholeStageCodegen (16) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #17 - WholeStageCodegen (15) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B6_LP,B6_CNT,B6_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #16 + WholeStageCodegen (10) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #17 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/explain.txt index 37387c2435..48ef976bdc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) : :- CometProject (30) : : +- CometBroadcastHashJoin (29) : : :- CometProject (24) @@ -19,42 +19,54 @@ TakeOrderedAndProject (43) : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : +- CometBroadcastExchange (28) : : +- CometProject (27) : : +- CometFilter (26) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (31) - +- ReusedExchange (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometNativeScan parquet spark_catalog.default.store (31) + +- CometBroadcastExchange (40) + +- CometProject (39) + +- CometFilter (38) + +- CometNativeScan parquet spark_catalog.default.item (37) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] @@ -73,9 +85,13 @@ Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] @@ -94,9 +110,12 @@ Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] @@ -119,9 +138,12 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] @@ -144,9 +166,12 @@ Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (26) CometFilter Input [2]: [d_date_sk#22, d_year#23] @@ -169,61 +194,80 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(31) CometNativeScan: `spark_catalog`.`default`.`store` +(31) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (32) CometFilter Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Condition : isnotnull(s_store_sk#24) -(33) CometBroadcastExchange +(33) CometProject Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#25, 16)) AS s_store_id#27, s_store_name#26] + +(34) CometBroadcastExchange +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] -(34) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight -(35) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] - -(36) ReusedExchange [Reuses operator id: 33] -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] - -(37) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Right output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight - -(38) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] - -(39) ColumnarToRow [codegen id : 1] -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] - -(40) HashAggregate [codegen id : 1] -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(36) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] + +(37) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(38) CometFilter +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) + +(39) CometProject +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#29, 16)) AS i_item_id#31, i_item_desc#30] + +(40) CometBroadcastExchange +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] + +(41) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight + +(42) CometProject +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] + +(43) CometHashAggregate +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum#30, sum#31, sum#32] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -(41) Exchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(44) CometExchange +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(42) HashAggregate [codegen id : 2] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(45) CometHashAggregate +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum(ss_quantity#5)#36, sum(sr_return_quantity#10)#37, sum(cs_quantity#14)#38] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum(ss_quantity#5)#36 AS store_sales_quantity#39, sum(sr_return_quantity#10)#37 AS store_returns_quantity#40, sum(cs_quantity#14)#38 AS catalog_sales_quantity#41] -(43) TakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] -Arguments: 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] +(46) CometTakeOrderedAndProject +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] + +(47) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/simplified.txt index 9d86a2fc9f..53a498d595 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/simplified.txt @@ -1,47 +1,49 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #6 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/explain.txt index de461c980e..fb45898a39 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/explain.txt @@ -1,28 +1,31 @@ == Physical Plan == TakeOrderedAndProject (19) +- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] @@ -63,41 +73,39 @@ Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isno (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(16) HashAggregate [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(17) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] -(18) HashAggregate [codegen id : 2] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(18) HashAggregate [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] +Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#11 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] (19) TakeOrderedAndProject Input [4]: [d_year#2, brand_id#14, brand#15, sum_agg#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/extended.txt new file mode 100644 index 0000000000..a0566bcef0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/extended.txt @@ -0,0 +1,21 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#1, i_brand_id#2 AS brand_id#3, i_brand#4 AS brand#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#7,17,2) AS sum_agg#8)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 16 out of 18 eligible operators (88%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/simplified.txt index cbc5dad6b1..efdf9da440 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/simplified.txt @@ -1,23 +1,21 @@ TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (1) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/explain.txt index 26cc3d472f..5901fb2c0e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/explain.txt @@ -1,68 +1,78 @@ == Physical Plan == -TakeOrderedAndProject (51) -+- * Project (50) - +- * BroadcastHashJoin Inner BuildRight (49) - :- * Project (43) - : +- * BroadcastHashJoin Inner BuildRight (42) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (18) - : : : +- * HashAggregate (17) - : : : +- Exchange (16) - : : : +- * HashAggregate (15) - : : : +- * ColumnarToRow (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (54) ++- * Project (53) + +- * BroadcastHashJoin Inner BuildRight (52) + :- * Project (46) + : +- * BroadcastHashJoin Inner BuildRight (45) + : :- * Project (39) + : : +- * BroadcastHashJoin Inner BuildRight (38) + : : :- * Filter (19) + : : : +- * HashAggregate (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- BroadcastExchange (35) - : : +- * Filter (34) - : : +- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * ColumnarToRow (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometFilter (20) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (19) - : : : +- ReusedExchange (21) - : : +- ReusedExchange (24) - : +- BroadcastExchange (41) - : +- * ColumnarToRow (40) - : +- CometFilter (39) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (38) - +- BroadcastExchange (48) - +- * ColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_returns` + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : : +- BroadcastExchange (37) + : : +- * Filter (36) + : : +- * HashAggregate (35) + : : +- * CometColumnarToRow (34) + : : +- CometColumnarExchange (33) + : : +- * HashAggregate (32) + : : +- * HashAggregate (31) + : : +- * CometColumnarToRow (30) + : : +- CometExchange (29) + : : +- CometHashAggregate (28) + : : +- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (21) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (20) + : : : +- ReusedExchange (22) + : : +- ReusedExchange (25) + : +- BroadcastExchange (44) + : +- * CometColumnarToRow (43) + : +- CometProject (42) + : +- CometFilter (41) + : +- CometNativeScan parquet spark_catalog.default.customer (40) + +- BroadcastExchange (51) + +- * CometColumnarToRow (50) + +- CometProject (49) + +- CometFilter (48) + +- CometNativeScan parquet spark_catalog.default.customer_address (47) + + +(1) CometNativeScan parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#4)] +PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_year#6] @@ -85,196 +95,216 @@ Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)))) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)) AS ca_state#9] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#7, ca_state#8] +Right output [2]: [ca_address_sk#7, ca_state#9] Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] - -(14) ColumnarToRow [codegen id : 1] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] +(14) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] -(15) HashAggregate [codegen id : 1] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -(16) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) HashAggregate [codegen id : 7] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(17) CometColumnarToRow [codegen id : 5] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] + +(18) HashAggregate [codegen id : 5] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] +Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#9 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] -(18) Filter [codegen id : 7] +(19) Filter [codegen id : 5] Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] Condition : isnotnull(ctr_total_return#14) -(19) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(20) CometNativeScan parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] -Arguments: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#18)] +PushedFilters: [IsNotNull(wr_returning_addr_sk)] +ReadSchema: struct -(20) CometFilter +(21) CometFilter Input [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] Condition : isnotnull(wr_returning_addr_sk#16) -(21) ReusedExchange [Reuses operator id: 6] +(22) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#19] -(22) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] Right output [1]: [d_date_sk#19] Arguments: [wr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight -(23) CometProject +(24) CometProject Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18, d_date_sk#19] Arguments: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17], [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] -(24) ReusedExchange [Reuses operator id: 11] +(25) ReusedExchange [Reuses operator id: 12] Output [2]: [ca_address_sk#20, ca_state#21] -(25) CometBroadcastHashJoin +(26) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] Right output [2]: [ca_address_sk#20, ca_state#21] Arguments: [wr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(26) CometProject +(27) CometProject Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, ca_address_sk#20, ca_state#21] Arguments: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21], [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] -(27) ColumnarToRow [codegen id : 2] -Input [3]: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] - -(28) HashAggregate [codegen id : 2] +(28) CometHashAggregate Input [3]: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] Keys [2]: [wr_returning_customer_sk#15, ca_state#21] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#17))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [wr_returning_customer_sk#15, ca_state#21, sum#23] -(29) Exchange -Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#23] -Arguments: hashpartitioning(wr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(29) CometExchange +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] +Arguments: hashpartitioning(wr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] -(30) HashAggregate [codegen id : 3] -Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#23] +(31) HashAggregate [codegen id : 1] +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] Keys [2]: [wr_returning_customer_sk#15, ca_state#21] Functions [1]: [sum(UnscaledValue(wr_return_amt#17))] Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#17))#11] -Results [2]: [ca_state#21 AS ctr_state#24, MakeDecimal(sum(UnscaledValue(wr_return_amt#17))#11,17,2) AS ctr_total_return#25] - -(31) HashAggregate [codegen id : 3] -Input [2]: [ctr_state#24, ctr_total_return#25] -Keys [1]: [ctr_state#24] -Functions [1]: [partial_avg(ctr_total_return#25)] -Aggregate Attributes [2]: [sum#26, count#27] -Results [3]: [ctr_state#24, sum#28, count#29] - -(32) Exchange -Input [3]: [ctr_state#24, sum#28, count#29] -Arguments: hashpartitioning(ctr_state#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(33) HashAggregate [codegen id : 4] -Input [3]: [ctr_state#24, sum#28, count#29] -Keys [1]: [ctr_state#24] -Functions [1]: [avg(ctr_total_return#25)] -Aggregate Attributes [1]: [avg(ctr_total_return#25)#30] -Results [2]: [(avg(ctr_total_return#25)#30 * 1.2) AS (avg(ctr_total_return) * 1.2)#31, ctr_state#24] - -(34) Filter [codegen id : 4] -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#31) - -(35) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] +Results [2]: [ca_state#21 AS ctr_state#23, MakeDecimal(sum(UnscaledValue(wr_return_amt#17))#11,17,2) AS ctr_total_return#24] + +(32) HashAggregate [codegen id : 1] +Input [2]: [ctr_state#23, ctr_total_return#24] +Keys [1]: [ctr_state#23] +Functions [1]: [partial_avg(ctr_total_return#24)] +Aggregate Attributes [2]: [sum#25, count#26] +Results [3]: [ctr_state#23, sum#27, count#28] + +(33) CometColumnarExchange +Input [3]: [ctr_state#23, sum#27, count#28] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(34) CometColumnarToRow [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] + +(35) HashAggregate [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] +Keys [1]: [ctr_state#23] +Functions [1]: [avg(ctr_total_return#24)] +Aggregate Attributes [1]: [avg(ctr_total_return#24)#29] +Results [2]: [(avg(ctr_total_return#24)#29 * 1.2) AS (avg(ctr_total_return) * 1.2)#30, ctr_state#23] + +(36) Filter [codegen id : 2] +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#30) + +(37) BroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] -(36) BroadcastHashJoin [codegen id : 7] +(38) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#24] +Right keys [1]: [ctr_state#23] Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) +Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#30) -(37) Project [codegen id : 7] +(39) Project [codegen id : 5] Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#31, ctr_state#24] +Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#30, ctr_state#23] + +(40) CometNativeScan parquet spark_catalog.default.customer +Output [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct -(38) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Arguments: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] +(41) CometFilter +Input [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Condition : (isnotnull(c_customer_sk#31) AND isnotnull(c_current_addr_sk#33)) -(39) CometFilter -Input [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_current_addr_sk#34)) +(42) CometProject +Input [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Arguments: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44], [c_customer_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#32, 16)) AS c_customer_id#45, c_current_addr_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#34, 10)) AS c_salutation#46, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#35, 20)) AS c_first_name#47, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#36, 30)) AS c_last_name#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#37, 1)) AS c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#42, 13)) AS c_login#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#43, 50)) AS c_email_address#51, c_last_review_date#44] -(40) ColumnarToRow [codegen id : 5] -Input [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] +(43) CometColumnarToRow [codegen id : 3] +Input [14]: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] -(41) BroadcastExchange -Input [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(44) BroadcastExchange +Input [14]: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -(42) BroadcastHashJoin [codegen id : 7] +(45) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#32] +Right keys [1]: [c_customer_sk#31] Join type: Inner Join condition: None -(43) Project [codegen id : 7] -Output [14]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] +(46) Project [codegen id : 5] +Output [14]: [ctr_total_return#14, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] +Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] -(44) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#46, ca_state#47] -Arguments: [ca_address_sk#46, ca_state#47] +(47) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#52, ca_state#53] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct -(45) CometFilter -Input [2]: [ca_address_sk#46, ca_state#47] -Condition : ((isnotnull(ca_state#47) AND (ca_state#47 = GA)) AND isnotnull(ca_address_sk#46)) +(48) CometFilter +Input [2]: [ca_address_sk#52, ca_state#53] +Condition : ((isnotnull(ca_state#53) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#53, 2)) = GA)) AND isnotnull(ca_address_sk#52)) -(46) CometProject -Input [2]: [ca_address_sk#46, ca_state#47] -Arguments: [ca_address_sk#46], [ca_address_sk#46] +(49) CometProject +Input [2]: [ca_address_sk#52, ca_state#53] +Arguments: [ca_address_sk#52], [ca_address_sk#52] -(47) ColumnarToRow [codegen id : 6] -Input [1]: [ca_address_sk#46] +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [ca_address_sk#52] -(48) BroadcastExchange -Input [1]: [ca_address_sk#46] +(51) BroadcastExchange +Input [1]: [ca_address_sk#52] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(49) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#46] +(52) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#33] +Right keys [1]: [ca_address_sk#52] Join type: Inner Join condition: None -(50) Project [codegen id : 7] -Output [13]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ctr_total_return#14] -Input [15]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ca_address_sk#46] +(53) Project [codegen id : 5] +Output [13]: [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] +Input [15]: [ctr_total_return#14, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ca_address_sk#52] -(51) TakeOrderedAndProject -Input [13]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ctr_total_return#14] -Arguments: 100, [c_customer_id#33 ASC NULLS FIRST, c_salutation#35 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, c_last_name#37 ASC NULLS FIRST, c_preferred_cust_flag#38 ASC NULLS FIRST, c_birth_day#39 ASC NULLS FIRST, c_birth_month#40 ASC NULLS FIRST, c_birth_year#41 ASC NULLS FIRST, c_birth_country#42 ASC NULLS FIRST, c_login#43 ASC NULLS FIRST, c_email_address#44 ASC NULLS FIRST, c_last_review_date#45 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ctr_total_return#14] +(54) TakeOrderedAndProject +Input [13]: [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] +Arguments: 100, [c_customer_id#45 ASC NULLS FIRST, c_salutation#46 ASC NULLS FIRST, c_first_name#47 ASC NULLS FIRST, c_last_name#48 ASC NULLS FIRST, c_preferred_cust_flag#49 ASC NULLS FIRST, c_birth_day#38 ASC NULLS FIRST, c_birth_month#39 ASC NULLS FIRST, c_birth_year#40 ASC NULLS FIRST, c_birth_country#41 ASC NULLS FIRST, c_login#50 ASC NULLS FIRST, c_email_address#51 ASC NULLS FIRST, c_last_review_date#44 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/extended.txt new file mode 100644 index 0000000000..bf988e5a3b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/extended.txt @@ -0,0 +1,62 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Filter + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wr_returning_customer_sk#1 AS ctr_customer_sk#2, ca_state#3 AS ctr_state#4, MakeDecimal(sum(UnscaledValue(wr_return_amt#5))#6,17,2) AS ctr_total_return#7)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- BroadcastExchange + : : +- Filter + : : +- HashAggregate + : : +- CometColumnarToRow + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_state#8 AS ctr_state#9, MakeDecimal(sum(UnscaledValue(wr_return_amt#10))#6,17,2) AS ctr_total_return#11)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 39 out of 55 eligible operators (70%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/simplified.txt index cf0645ac36..b1bf929a71 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - WholeStageCodegen (7) + WholeStageCodegen (5) Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] BroadcastHashJoin [c_current_addr_sk,ca_address_sk] Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] @@ -8,61 +8,60 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] Filter [ctr_total_return] HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (1) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometColumnarToRow + InputAdapter + CometExchange [wr_returning_customer_sk,ca_state] #1 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #4 - WholeStageCodegen (4) + WholeStageCodegen (2) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (3) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (2) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - ReusedExchange [ca_address_sk,ca_state] #3 + CometColumnarToRow + InputAdapter + CometColumnarExchange [ctr_state] #5 + WholeStageCodegen (1) + HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] + HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [wr_returning_customer_sk,ca_state] #6 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 InputAdapter BroadcastExchange #7 - WholeStageCodegen (5) - ColumnarToRow + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] InputAdapter BroadcastExchange #8 - WholeStageCodegen (6) - ColumnarToRow + WholeStageCodegen (4) + CometColumnarToRow InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/explain.txt index 0167b4b1a4..03dbef5ff2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/explain.txt @@ -1,79 +1,120 @@ == Physical Plan == -* ColumnarToRow (62) -+- CometSort (61) - +- CometColumnarExchange (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * BroadcastHashJoin Inner BuildRight (53) +* CometColumnarToRow (96) ++- CometSort (95) + +- CometColumnarExchange (94) + +- * Project (93) + +- * BroadcastHashJoin Inner BuildRight (92) + :- * Project (78) + : +- * BroadcastHashJoin Inner BuildRight (77) + : :- * BroadcastHashJoin Inner BuildRight (63) : : :- * Project (49) : : : +- * BroadcastHashJoin Inner BuildRight (48) : : : :- * BroadcastHashJoin Inner BuildRight (32) : : : : :- * HashAggregate (16) - : : : : : +- Exchange (15) - : : : : : +- * HashAggregate (14) - : : : : : +- * ColumnarToRow (13) + : : : : : +- * CometColumnarToRow (15) + : : : : : +- CometExchange (14) + : : : : : +- CometHashAggregate (13) : : : : : +- CometProject (12) : : : : : +- CometBroadcastHashJoin (11) : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) : : : : +- BroadcastExchange (31) : : : : +- * HashAggregate (30) - : : : : +- Exchange (29) - : : : : +- * HashAggregate (28) - : : : : +- * ColumnarToRow (27) + : : : : +- * CometColumnarToRow (29) + : : : : +- CometExchange (28) + : : : : +- CometHashAggregate (27) : : : : +- CometProject (26) : : : : +- CometBroadcastHashJoin (25) : : : : :- CometProject (23) : : : : : +- CometBroadcastHashJoin (22) : : : : : :- CometFilter (18) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (17) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (17) : : : : : +- CometBroadcastExchange (21) : : : : : +- CometFilter (20) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) : : : : +- ReusedExchange (24) : : : +- BroadcastExchange (47) : : : +- * HashAggregate (46) - : : : +- Exchange (45) - : : : +- * HashAggregate (44) - : : : +- * ColumnarToRow (43) + : : : +- * CometColumnarToRow (45) + : : : +- CometExchange (44) + : : : +- CometHashAggregate (43) : : : +- CometProject (42) : : : +- CometBroadcastHashJoin (41) : : : :- CometProject (39) : : : : +- CometBroadcastHashJoin (38) : : : : :- CometFilter (34) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (33) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (33) : : : : +- CometBroadcastExchange (37) : : : : +- CometFilter (36) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (35) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (35) : : : +- ReusedExchange (40) - : : +- BroadcastExchange (52) - : : +- * HashAggregate (51) - : : +- ReusedExchange (50) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- BroadcastExchange (62) + : : +- * HashAggregate (61) + : : +- * CometColumnarToRow (60) + : : +- CometExchange (59) + : : +- CometHashAggregate (58) + : : +- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- BroadcastExchange (76) + : +- * HashAggregate (75) + : +- * CometColumnarToRow (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + +- BroadcastExchange (91) + +- * HashAggregate (90) + +- * CometColumnarToRow (89) + +- CometExchange (88) + +- CometHashAggregate (87) + +- CometProject (86) + +- CometBroadcastHashJoin (85) + :- CometProject (83) + : +- CometBroadcastHashJoin (82) + : :- CometFilter (80) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (79) + : +- ReusedExchange (81) + +- ReusedExchange (84) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] @@ -92,9 +133,12 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ca_address_sk#7, ca_county#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] +ReadSchema: struct (9) CometFilter Input [2]: [ca_address_sk#7, ca_county#8] @@ -113,229 +157,394 @@ Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] -(13) ColumnarToRow [codegen id : 1] -Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] - -(14) HashAggregate [codegen id : 1] +(13) CometHashAggregate Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#9] -Results [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -(15) Exchange -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(14) CometExchange +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) HashAggregate [codegen id : 12] -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] +(15) CometColumnarToRow [codegen id : 6] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] + +(16) HashAggregate [codegen id : 6] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#11] -Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS store_sales#12] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#10] +Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS store_sales#11] -(17) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Arguments: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] +(17) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#14)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct (18) CometFilter -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_addr_sk#13) +Input [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Condition : isnotnull(ss_addr_sk#12) -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [d_date_sk#16, d_year#17, d_qoy#18] +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Condition : ((((isnotnull(d_qoy#18) AND isnotnull(d_year#17)) AND (d_qoy#18 = 2)) AND (d_year#17 = 2000)) AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Condition : ((((isnotnull(d_qoy#17) AND isnotnull(d_year#16)) AND (d_qoy#17 = 2)) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15)) (21) CometBroadcastExchange -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [d_date_sk#16, d_year#17, d_qoy#18] +Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [d_date_sk#15, d_year#16, d_qoy#17] (22) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Right output [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [ss_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight +Left output [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Right output [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight (23) CometProject -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18], [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18] +Input [6]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17], [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17] (24) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#19, ca_county#20] +Output [2]: [ca_address_sk#18, ca_county#19] (25) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18] -Right output [2]: [ca_address_sk#19, ca_county#20] -Arguments: [ss_addr_sk#13], [ca_address_sk#19], Inner, BuildRight +Left output [4]: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17] +Right output [2]: [ca_address_sk#18, ca_county#19] +Arguments: [ss_addr_sk#12], [ca_address_sk#18], Inner, BuildRight (26) CometProject -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_address_sk#19, ca_county#20] -Arguments: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20], [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] - -(27) ColumnarToRow [codegen id : 2] -Input [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] - -(28) HashAggregate [codegen id : 2] -Input [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] - -(29) Exchange -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Arguments: hashpartitioning(ca_county#20, d_qoy#18, d_year#17, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(30) HashAggregate [codegen id : 3] -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#14))#11] -Results [2]: [ca_county#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#14))#11,17,2) AS store_sales#23] +Input [6]: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_address_sk#18, ca_county#19] +Arguments: [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19], [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19] + +(27) CometHashAggregate +Input [4]: [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19] +Keys [3]: [ca_county#19, d_qoy#17, d_year#16] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#13))] + +(28) CometExchange +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] +Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] + +(30) HashAggregate [codegen id : 1] +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] +Keys [3]: [ca_county#19, d_qoy#17, d_year#16] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#13))#10] +Results [2]: [ca_county#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#13))#10,17,2) AS store_sales#21] (31) BroadcastExchange -Input [2]: [ca_county#20, store_sales#23] +Input [2]: [ca_county#19, store_sales#21] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(32) BroadcastHashJoin [codegen id : 12] +(32) BroadcastHashJoin [codegen id : 6] Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#20] +Right keys [1]: [ca_county#19] Join type: Inner Join condition: None -(33) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Arguments: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(33) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct (34) CometFilter -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_addr_sk#24) +Input [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_addr_sk#22) -(35) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [d_date_sk#27, d_year#28, d_qoy#29] +(35) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (36) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +Input [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Condition : ((((isnotnull(d_qoy#27) AND isnotnull(d_year#26)) AND (d_qoy#27 = 3)) AND (d_year#26 = 2000)) AND isnotnull(d_date_sk#25)) (37) CometBroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [d_date_sk#27, d_year#28, d_qoy#29] +Input [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [d_date_sk#25, d_year#26, d_qoy#27] (38) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight +Left output [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Right output [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight (39) CometProject -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29], [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29] +Input [6]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27], [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27] (40) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#30, ca_county#31] +Output [2]: [ca_address_sk#28, ca_county#29] (41) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29] -Right output [2]: [ca_address_sk#30, ca_county#31] -Arguments: [ss_addr_sk#24], [ca_address_sk#30], Inner, BuildRight +Left output [4]: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27] +Right output [2]: [ca_address_sk#28, ca_county#29] +Arguments: [ss_addr_sk#22], [ca_address_sk#28], Inner, BuildRight (42) CometProject -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_address_sk#30, ca_county#31] -Arguments: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31], [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] - -(43) ColumnarToRow [codegen id : 4] -Input [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] - -(44) HashAggregate [codegen id : 4] -Input [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum#32] -Results [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] - -(45) Exchange -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(46) HashAggregate [codegen id : 5] -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#25))#11] -Results [2]: [ca_county#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#25))#11,17,2) AS store_sales#34] +Input [6]: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_address_sk#28, ca_county#29] +Arguments: [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29], [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29] -(47) BroadcastExchange -Input [2]: [ca_county#31, store_sales#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] +(43) CometHashAggregate +Input [4]: [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29] +Keys [3]: [ca_county#29, d_qoy#27, d_year#26] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] -(48) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#20] -Right keys [1]: [ca_county#31] -Join type: Inner -Join condition: None +(44) CometExchange +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] +Arguments: hashpartitioning(ca_county#29, d_qoy#27, d_year#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(49) Project [codegen id : 12] -Output [5]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34] -Input [7]: [ca_county#8, d_year#5, store_sales#12, ca_county#20, store_sales#23, ca_county#31, store_sales#34] +(45) CometColumnarToRow [codegen id : 2] +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] -(50) ReusedExchange [Reuses operator id: 15] -Output [4]: [ca_county#35, d_qoy#36, d_year#37, sum#38] +(46) HashAggregate [codegen id : 2] +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] +Keys [3]: [ca_county#29, d_qoy#27, d_year#26] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#23))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#23))#10] +Results [2]: [ca_county#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#23))#10,17,2) AS store_sales#31] -(51) HashAggregate [codegen id : 7] -Input [4]: [ca_county#35, d_qoy#36, d_year#37, sum#38] -Keys [3]: [ca_county#35, d_qoy#36, d_year#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#40] -Results [2]: [ca_county#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#40,17,2) AS web_sales#41] +(47) BroadcastExchange +Input [2]: [ca_county#29, store_sales#31] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(52) BroadcastExchange -Input [2]: [ca_county#35, web_sales#41] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] +(48) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#19] +Right keys [1]: [ca_county#29] +Join type: Inner +Join condition: None -(53) BroadcastHashJoin [codegen id : 12] +(49) Project [codegen id : 6] +Output [5]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31] +Input [7]: [ca_county#8, d_year#5, store_sales#11, ca_county#19, store_sales#21, ca_county#29, store_sales#31] + +(50) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(51) CometFilter +Input [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : isnotnull(ws_bill_addr_sk#32) + +(52) ReusedExchange [Reuses operator id: 5] +Output [3]: [d_date_sk#35, d_year#36, d_qoy#37] + +(53) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [3]: [d_date_sk#35, d_year#36, d_qoy#37] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight + +(54) CometProject +Input [6]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35, d_year#36, d_qoy#37] +Arguments: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37], [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37] + +(55) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#38, ca_county#39] + +(56) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37] +Right output [2]: [ca_address_sk#38, ca_county#39] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#38], Inner, BuildRight + +(57) CometProject +Input [6]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_address_sk#38, ca_county#39] +Arguments: [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39], [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39] + +(58) CometHashAggregate +Input [4]: [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39] +Keys [3]: [ca_county#39, d_qoy#37, d_year#36] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] + +(59) CometExchange +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] +Arguments: hashpartitioning(ca_county#39, d_qoy#37, d_year#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(60) CometColumnarToRow [codegen id : 3] +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] + +(61) HashAggregate [codegen id : 3] +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] +Keys [3]: [ca_county#39, d_qoy#37, d_year#36] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#41] +Results [2]: [ca_county#39, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#41,17,2) AS web_sales#42] + +(62) BroadcastExchange +Input [2]: [ca_county#39, web_sales#42] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(63) BroadcastHashJoin [codegen id : 6] Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#35] +Right keys [1]: [ca_county#39] Join type: Inner Join condition: None -(54) ReusedExchange [Reuses operator id: 31] -Output [2]: [ca_county#42, web_sales#43] - -(55) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#35] -Right keys [1]: [ca_county#42] +(64) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#45)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(65) CometFilter +Input [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Condition : isnotnull(ws_bill_addr_sk#43) + +(66) ReusedExchange [Reuses operator id: 21] +Output [3]: [d_date_sk#46, d_year#47, d_qoy#48] + +(67) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Right output [3]: [d_date_sk#46, d_year#47, d_qoy#48] +Arguments: [ws_sold_date_sk#45], [d_date_sk#46], Inner, BuildRight + +(68) CometProject +Input [6]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45, d_date_sk#46, d_year#47, d_qoy#48] +Arguments: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48], [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48] + +(69) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#49, ca_county#50] + +(70) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48] +Right output [2]: [ca_address_sk#49, ca_county#50] +Arguments: [ws_bill_addr_sk#43], [ca_address_sk#49], Inner, BuildRight + +(71) CometProject +Input [6]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_address_sk#49, ca_county#50] +Arguments: [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50], [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50] + +(72) CometHashAggregate +Input [4]: [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50] +Keys [3]: [ca_county#50, d_qoy#48, d_year#47] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#44))] + +(73) CometExchange +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] +Arguments: hashpartitioning(ca_county#50, d_qoy#48, d_year#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(74) CometColumnarToRow [codegen id : 4] +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] + +(75) HashAggregate [codegen id : 4] +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] +Keys [3]: [ca_county#50, d_qoy#48, d_year#47] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#44))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#44))#41] +Results [2]: [ca_county#50, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#44))#41,17,2) AS web_sales#52] + +(76) BroadcastExchange +Input [2]: [ca_county#50, web_sales#52] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] + +(77) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#39] +Right keys [1]: [ca_county#50] Join type: Inner -Join condition: (CASE WHEN (web_sales#41 > 0.00) THEN (web_sales#43 / web_sales#41) END > CASE WHEN (store_sales#12 > 0.00) THEN (store_sales#23 / store_sales#12) END) - -(56) Project [codegen id : 12] -Output [8]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#35, web_sales#41, web_sales#43] -Input [9]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#35, web_sales#41, ca_county#42, web_sales#43] - -(57) ReusedExchange [Reuses operator id: 47] -Output [2]: [ca_county#44, web_sales#45] - -(58) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#35] -Right keys [1]: [ca_county#44] +Join condition: (CASE WHEN (web_sales#42 > 0.00) THEN (web_sales#52 / web_sales#42) END > CASE WHEN (store_sales#11 > 0.00) THEN (store_sales#21 / store_sales#11) END) + +(78) Project [codegen id : 6] +Output [8]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, web_sales#52] +Input [9]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, ca_county#50, web_sales#52] + +(79) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#55)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(80) CometFilter +Input [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Condition : isnotnull(ws_bill_addr_sk#53) + +(81) ReusedExchange [Reuses operator id: 37] +Output [3]: [d_date_sk#56, d_year#57, d_qoy#58] + +(82) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Right output [3]: [d_date_sk#56, d_year#57, d_qoy#58] +Arguments: [ws_sold_date_sk#55], [d_date_sk#56], Inner, BuildRight + +(83) CometProject +Input [6]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55, d_date_sk#56, d_year#57, d_qoy#58] +Arguments: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58], [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58] + +(84) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#59, ca_county#60] + +(85) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58] +Right output [2]: [ca_address_sk#59, ca_county#60] +Arguments: [ws_bill_addr_sk#53], [ca_address_sk#59], Inner, BuildRight + +(86) CometProject +Input [6]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_address_sk#59, ca_county#60] +Arguments: [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60], [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60] + +(87) CometHashAggregate +Input [4]: [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60] +Keys [3]: [ca_county#60, d_qoy#58, d_year#57] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#54))] + +(88) CometExchange +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] +Arguments: hashpartitioning(ca_county#60, d_qoy#58, d_year#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(89) CometColumnarToRow [codegen id : 5] +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] + +(90) HashAggregate [codegen id : 5] +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] +Keys [3]: [ca_county#60, d_qoy#58, d_year#57] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#54))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#54))#41] +Results [2]: [ca_county#60, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#54))#41,17,2) AS web_sales#62] + +(91) BroadcastExchange +Input [2]: [ca_county#60, web_sales#62] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] + +(92) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#39] +Right keys [1]: [ca_county#60] Join type: Inner -Join condition: (CASE WHEN (web_sales#43 > 0.00) THEN (web_sales#45 / web_sales#43) END > CASE WHEN (store_sales#23 > 0.00) THEN (store_sales#34 / store_sales#23) END) +Join condition: (CASE WHEN (web_sales#52 > 0.00) THEN (web_sales#62 / web_sales#52) END > CASE WHEN (store_sales#21 > 0.00) THEN (store_sales#31 / store_sales#21) END) -(59) Project [codegen id : 12] -Output [6]: [ca_county#8, d_year#5, (web_sales#43 / web_sales#41) AS web_q1_q2_increase#46, (store_sales#23 / store_sales#12) AS store_q1_q2_increase#47, (web_sales#45 / web_sales#43) AS web_q2_q3_increase#48, (store_sales#34 / store_sales#23) AS store_q2_q3_increase#49] -Input [10]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#35, web_sales#41, web_sales#43, ca_county#44, web_sales#45] +(93) Project [codegen id : 6] +Output [6]: [ca_county#8, d_year#5, (web_sales#52 / web_sales#42) AS web_q1_q2_increase#63, (store_sales#21 / store_sales#11) AS store_q1_q2_increase#64, (web_sales#62 / web_sales#52) AS web_q2_q3_increase#65, (store_sales#31 / store_sales#21) AS store_q2_q3_increase#66] +Input [10]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, web_sales#52, ca_county#60, web_sales#62] -(60) CometColumnarExchange -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +(94) CometColumnarExchange +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] -(61) CometSort -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49] -Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49], [ca_county#8 ASC NULLS FIRST] +(95) CometSort +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66], [ca_county#8 ASC NULLS FIRST] -(62) ColumnarToRow [codegen id : 13] -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49] +(96) CometColumnarToRow [codegen id : 7] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/extended.txt new file mode 100644 index 0000000000..949bdf19ba --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/extended.txt @@ -0,0 +1,114 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- BroadcastHashJoin + : : : : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_county#1, d_year#2, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#4,17,2) AS store_sales#5)] + : : : : : +- CometColumnarToRow + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- BroadcastExchange + : : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_county#6, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#4,17,2) AS store_sales#8)] + : : : : +- CometColumnarToRow + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- BroadcastExchange + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_county#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#10))#4,17,2) AS store_sales#11)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_county#12, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#13))#14,17,2) AS web_sales#15)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- BroadcastExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_county#16, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#17))#14,17,2) AS web_sales#18)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_county#19, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#20))#14,17,2) AS web_sales#21)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 86 out of 105 eligible operators (81%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/simplified.txt index 069817737f..34f848f327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ -WholeStageCodegen (13) - ColumnarToRow +WholeStageCodegen (7) + CometColumnarToRow InputAdapter CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] CometColumnarExchange [ca_county] #1 - WholeStageCodegen (12) + WholeStageCodegen (6) Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] Project [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] @@ -13,71 +13,103 @@ WholeStageCodegen (13) BroadcastHashJoin [ca_county,ca_county] BroadcastHashJoin [ca_county,ca_county] HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #2 - WholeStageCodegen (1) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [ca_address_sk,ca_county] #4 - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #2 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [ca_address_sk,ca_county] #4 + CometFilter [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #5 - WholeStageCodegen (3) + WholeStageCodegen (1) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #6 - WholeStageCodegen (2) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #4 + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #6 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter BroadcastExchange #8 - WholeStageCodegen (5) + WholeStageCodegen (2) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #9 - WholeStageCodegen (4) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #4 + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #9 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter BroadcastExchange #11 - WholeStageCodegen (7) + WholeStageCodegen (3) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - ReusedExchange [ca_county,d_qoy,d_year,sum] #2 + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #12 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter - ReusedExchange [ca_county,web_sales] #5 + BroadcastExchange #13 + WholeStageCodegen (4) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #14 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter - ReusedExchange [ca_county,web_sales] #8 + BroadcastExchange #15 + WholeStageCodegen (5) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #16 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/explain.txt index 8514993cd1..79e86e8779 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/explain.txt @@ -1,52 +1,54 @@ == Physical Plan == -* HashAggregate (35) -+- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- BroadcastExchange (23) - : +- * Filter (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (10) - : +- CometBroadcastExchange (15) - : +- CometProject (14) - : +- CometFilter (13) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - +- BroadcastExchange (30) - +- * ColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +* HashAggregate (30) ++- * CometColumnarToRow (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] @@ -69,125 +71,105 @@ Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -(9) ColumnarToRow [codegen id : 4] -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] - -(10) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Arguments: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct -(11) CometFilter +(10) CometFilter Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Condition : isnotnull(cs_item_sk#6) -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] +ReadSchema: struct -(13) CometFilter +(12) CometFilter Input [2]: [d_date_sk#9, d_date#10] Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) -(14) CometProject +(13) CometProject Input [2]: [d_date_sk#9, d_date#10] Arguments: [d_date_sk#9], [d_date_sk#9] -(15) CometBroadcastExchange +(14) CometBroadcastExchange Input [1]: [d_date_sk#9] Arguments: [d_date_sk#9] -(16) CometBroadcastHashJoin +(15) CometBroadcastHashJoin Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Right output [1]: [d_date_sk#9] Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(17) CometProject +(16) CometProject Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] -(18) ColumnarToRow [codegen id : 1] -Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] - -(19) HashAggregate [codegen id : 1] +(17) CometHashAggregate Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] Keys [1]: [cs_item_sk#6] Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [cs_item_sk#6, sum#13, count#14] -(20) Exchange -Input [3]: [cs_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(18) CometExchange +Input [3]: [cs_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 2] -Input [3]: [cs_item_sk#6, sum#13, count#14] +(19) CometHashAggregate +Input [3]: [cs_item_sk#6, sum#11, count#12] Keys [1]: [cs_item_sk#6] Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(cs_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] - -(22) Filter [codegen id : 2] -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#16) -(23) BroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=2] +(20) CometFilter +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [cs_item_sk#6] -Join type: Inner -Join condition: (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#16) +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] -(25) Project [codegen id : 4] -Output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] +(22) CometBroadcastHashJoin +Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight -(26) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17, d_date#18] +(23) CometProject +Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] -(27) CometFilter -Input [2]: [d_date_sk#17, d_date#18] -Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-01-27)) AND (d_date#18 <= 2000-04-26)) AND isnotnull(d_date_sk#17)) +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#14] -(28) CometProject -Input [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17], [d_date_sk#17] +(25) CometBroadcastHashJoin +Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight -(29) ColumnarToRow [codegen id : 3] -Input [1]: [d_date_sk#17] +(26) CometProject +Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#14] +Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] -(30) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(31) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 4] -Output [1]: [cs_ext_discount_amt#2] -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#17] - -(33) HashAggregate [codegen id : 4] +(27) CometHashAggregate Input [1]: [cs_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#19] -Results [1]: [sum#20] -(34) Exchange -Input [1]: [sum#20] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [1]: [sum#15] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum#15] -(35) HashAggregate [codegen id : 5] -Input [1]: [sum#20] +(30) HashAggregate [codegen id : 1] +Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#21] -Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#21,17,2) AS excess discount amount#22] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#16] +Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#16,17,2) AS excess discount amount#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/extended.txt new file mode 100644 index 0000000000..07c610a45c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/extended.txt @@ -0,0 +1,35 @@ + HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#1))#2,17,2) AS excess discount amount#3)] ++- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 31 out of 32 eligible operators (96%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/simplified.txt index 79e7290c35..769b514263 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/simplified.txt @@ -1,47 +1,32 @@ -WholeStageCodegen (5) +WholeStageCodegen (1) HashAggregate [sum] [sum(UnscaledValue(cs_ext_discount_amt)),excess discount amount,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [cs_ext_discount_amt] [sum,sum] - Project [cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_discount_amt,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] - ColumnarToRow - InputAdapter - CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk] #2 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [(1.3 * avg(cs_ext_discount_amt))] - HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count] - InputAdapter - Exchange [cs_item_sk] #4 - WholeStageCodegen (1) - HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_item_sk,cs_ext_discount_amt] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [cs_ext_discount_amt] [sum] + CometProject [cs_ext_discount_amt] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk,(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 + CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] + CometExchange [cs_item_sk] #4 + CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] + CometProject [cs_item_sk,cs_ext_discount_amt] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/explain.txt index f6b49a4d96..d4c6665cba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/explain.txt @@ -1,67 +1,88 @@ == Physical Plan == -TakeOrderedAndProject (50) -+- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- Union (46) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometProject (33) - : : : +- CometBroadcastHashJoin (32) - : : : :- CometFilter (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : : : +- ReusedExchange (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- * HashAggregate (45) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (64) ++- * HashAggregate (63) + +- * CometColumnarToRow (62) + +- CometColumnarExchange (61) + +- * HashAggregate (60) + +- Union (59) + :- * HashAggregate (28) + : +- * CometColumnarToRow (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- * HashAggregate (43) + : +- * CometColumnarToRow (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- * HashAggregate (58) + +- * CometColumnarToRow (57) + +- CometExchange (56) + +- CometHashAggregate (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometProject (48) + : : +- CometBroadcastHashJoin (47) + : : :- CometFilter (45) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (44) + : : +- ReusedExchange (46) + : +- ReusedExchange (49) + +- ReusedExchange (52) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] @@ -84,9 +105,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] @@ -109,21 +133,27 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [i_item_sk#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#10, i_manufact_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometNativeScan: `spark_catalog`.`default`.`item` +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_category#12, i_manufact_id#13] -Arguments: [i_category#12, i_manufact_id#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category)] +ReadSchema: struct (18) CometFilter Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) +Condition : (isnotnull(i_category#12) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#12, 50)) = Electronics )) (19) CometProject Input [2]: [i_category#12, i_manufact_id#13] @@ -151,123 +181,183 @@ Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] -(25) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] - -(26) HashAggregate [codegen id : 1] +(25) CometHashAggregate Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] Keys [1]: [i_manufact_id#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_manufact_id#11, sum#15] -(27) Exchange -Input [2]: [i_manufact_id#11, sum#15] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(26) CometExchange +Input [2]: [i_manufact_id#11, sum#14] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 2] -Input [2]: [i_manufact_id#11, sum#15] +(27) CometColumnarToRow [codegen id : 1] +Input [2]: [i_manufact_id#11, sum#14] + +(28) HashAggregate [codegen id : 1] +Input [2]: [i_manufact_id#11, sum#14] Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#15] +Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#15,17,2) AS total_sales#16] -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#22] +Output [1]: [d_date_sk#21] (32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] (34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#23] +Output [1]: [ca_address_sk#22] (35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#23] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#23], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight (36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] (37) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#24, i_manufact_id#25] +Output [2]: [i_item_sk#23, i_manufact_id#24] (38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#24, i_manufact_id#25] -Arguments: [cs_item_sk#19], [i_item_sk#24], Inner, BuildRight +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_manufact_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight (39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_manufact_id#25] -Arguments: [cs_ext_sales_price#20, i_manufact_id#25], [cs_ext_sales_price#20, i_manufact_id#25] - -(40) ColumnarToRow [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_manufact_id#25] - -(41) HashAggregate [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Keys [1]: [i_manufact_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_manufact_id#25, sum#27] - -(42) Exchange -Input [2]: [i_manufact_id#25, sum#27] -Arguments: hashpartitioning(i_manufact_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(43) HashAggregate [codegen id : 4] -Input [2]: [i_manufact_id#25, sum#27] -Keys [1]: [i_manufact_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_manufact_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(44) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_manufact_id#30, sum#31] - -(45) HashAggregate [codegen id : 6] -Input [2]: [i_manufact_id#30, sum#31] -Keys [1]: [i_manufact_id#30] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#33] -Results [2]: [i_manufact_id#30, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#33,17,2) AS total_sales#34] - -(46) Union - -(47) HashAggregate [codegen id : 7] -Input [2]: [i_manufact_id#11, total_sales#17] +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_manufact_id#24] +Arguments: [cs_ext_sales_price#19, i_manufact_id#24], [cs_ext_sales_price#19, i_manufact_id#24] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_manufact_id#24] +Keys [1]: [i_manufact_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] + +(41) CometExchange +Input [2]: [i_manufact_id#24, sum#25] +Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [2]: [i_manufact_id#24, sum#25] + +(43) HashAggregate [codegen id : 2] +Input [2]: [i_manufact_id#24, sum#25] +Keys [1]: [i_manufact_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#19))#26] +Results [2]: [i_manufact_id#24, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#19))#26,17,2) AS total_sales#27] + +(44) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#31)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(45) CometFilter +Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Condition : (isnotnull(ws_bill_addr_sk#29) AND isnotnull(ws_item_sk#28)) + +(46) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#32] + +(47) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#31], [d_date_sk#32], Inner, BuildRight + +(48) CometProject +Input [5]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31, d_date_sk#32] +Arguments: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30], [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] + +(49) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#33] + +(50) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] +Right output [1]: [ca_address_sk#33] +Arguments: [ws_bill_addr_sk#29], [ca_address_sk#33], Inner, BuildRight + +(51) CometProject +Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ca_address_sk#33] +Arguments: [ws_item_sk#28, ws_ext_sales_price#30], [ws_item_sk#28, ws_ext_sales_price#30] + +(52) ReusedExchange [Reuses operator id: 22] +Output [2]: [i_item_sk#34, i_manufact_id#35] + +(53) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#28, ws_ext_sales_price#30] +Right output [2]: [i_item_sk#34, i_manufact_id#35] +Arguments: [ws_item_sk#28], [i_item_sk#34], Inner, BuildRight + +(54) CometProject +Input [4]: [ws_item_sk#28, ws_ext_sales_price#30, i_item_sk#34, i_manufact_id#35] +Arguments: [ws_ext_sales_price#30, i_manufact_id#35], [ws_ext_sales_price#30, i_manufact_id#35] + +(55) CometHashAggregate +Input [2]: [ws_ext_sales_price#30, i_manufact_id#35] +Keys [1]: [i_manufact_id#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#30))] + +(56) CometExchange +Input [2]: [i_manufact_id#35, sum#36] +Arguments: hashpartitioning(i_manufact_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(57) CometColumnarToRow [codegen id : 3] +Input [2]: [i_manufact_id#35, sum#36] + +(58) HashAggregate [codegen id : 3] +Input [2]: [i_manufact_id#35, sum#36] +Keys [1]: [i_manufact_id#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#30))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#30))#37] +Results [2]: [i_manufact_id#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#30))#37,17,2) AS total_sales#38] + +(59) Union + +(60) HashAggregate [codegen id : 4] +Input [2]: [i_manufact_id#11, total_sales#16] Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [i_manufact_id#11, sum#37, isEmpty#38] +Functions [1]: [partial_sum(total_sales#16)] +Aggregate Attributes [2]: [sum#39, isEmpty#40] +Results [3]: [i_manufact_id#11, sum#41, isEmpty#42] + +(61) CometColumnarExchange +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(48) Exchange -Input [3]: [i_manufact_id#11, sum#37, isEmpty#38] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(62) CometColumnarToRow [codegen id : 5] +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] -(49) HashAggregate [codegen id : 8] -Input [3]: [i_manufact_id#11, sum#37, isEmpty#38] +(63) HashAggregate [codegen id : 5] +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#39] -Results [2]: [i_manufact_id#11, sum(total_sales#17)#39 AS total_sales#40] +Functions [1]: [sum(total_sales#16)] +Aggregate Attributes [1]: [sum(total_sales#16)#43] +Results [2]: [i_manufact_id#11, sum(total_sales#16)#43 AS total_sales#44] -(50) TakeOrderedAndProject -Input [2]: [i_manufact_id#11, total_sales#40] -Arguments: 100, [total_sales#40 ASC NULLS FIRST], [i_manufact_id#11, total_sales#40] +(64) TakeOrderedAndProject +Input [2]: [i_manufact_id#11, total_sales#44] +Arguments: 100, [total_sales#44 ASC NULLS FIRST], [i_manufact_id#11, total_sales#44] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/extended.txt new file mode 100644 index 0000000000..208a169e44 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/extended.txt @@ -0,0 +1,92 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_manufact_id#1, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#3,17,2) AS total_sales#4)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_manufact_id#5, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#6))#7,17,2) AS total_sales#8)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_manufact_id#9, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#10))#11,17,2) AS total_sales#12)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 79 out of 86 eligible operators (91%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/simplified.txt index 363d49dbb0..5275a3be76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/simplified.txt @@ -1,64 +1,74 @@ TakeOrderedAndProject [total_sales,i_manufact_id] - WholeStageCodegen (8) + WholeStageCodegen (5) HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (7) - HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #2 - WholeStageCodegen (1) - HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_manufact_id] #5 - CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #6 - CometProject [i_manufact_id] - CometFilter [i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact_id] - WholeStageCodegen (4) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #7 - WholeStageCodegen (3) - HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_manufact_id] #5 - WholeStageCodegen (6) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - ReusedExchange [i_manufact_id,sum] #2 + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_manufact_id] #1 + WholeStageCodegen (4) + HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_manufact_id,sum] + CometProject [ss_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_manufact_id] #5 + CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] + CometFilter [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [i_manufact_id] #6 + CometProject [i_manufact_id] + CometFilter [i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact_id] + WholeStageCodegen (2) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] + CometProject [cs_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 + WholeStageCodegen (3) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] + CometProject [ws_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/explain.txt index 4d010531bc..8262469d45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,35 +15,43 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] @@ -66,9 +74,12 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#9, s_county#10] @@ -91,13 +102,16 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -134,35 +148,42 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(25) CometNativeScan: `spark_catalog`.`default`.`customer` +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (26) CometFilter Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Condition : isnotnull(c_customer_sk#17) -(27) CometBroadcastExchange +(27) CometProject Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] -(28) CometBroadcastHashJoin +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/simplified.txt index bf05de8c6e..66a94b088b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] @@ -16,19 +16,20 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/explain.txt index aae420dfb5..3b80618463 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/explain.txt @@ -1,60 +1,77 @@ == Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] @@ -86,12 +103,15 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#13] @@ -105,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -118,92 +138,132 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct -(21) BroadcastHashJoin [codegen id : 5] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None -(22) Filter [codegen id : 5] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(23) Project [codegen id : 5] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_state#16] -Arguments: [ca_address_sk#15, ca_state#16] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : isnotnull(ca_address_sk#15) +(31) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] -(26) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#15, ca_state#16] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] -(27) BroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(28) BroadcastHashJoin [codegen id : 5] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(29) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#16] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#16] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(31) CometFilter -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Condition : isnotnull(cd_demo_sk#17) +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(33) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(34) BroadcastHashJoin [codegen id : 5] +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None -(35) Project [codegen id : 5] -Output [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Input [8]: [c_current_cdemo_sk#4, ca_state#16, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(36) HashAggregate [codegen id : 5] -Input [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#20), partial_max(cd_dep_count#20), partial_avg(cd_dep_count#20), partial_min(cd_dep_employed_count#21), partial_max(cd_dep_employed_count#21), partial_avg(cd_dep_employed_count#21), partial_min(cd_dep_college_count#22), partial_max(cd_dep_college_count#22), partial_avg(cd_dep_college_count#22)] -Aggregate Attributes [13]: [count#23, min#24, max#25, sum#26, count#27, min#28, max#29, sum#30, count#31, min#32, max#33, sum#34, count#35] -Results [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] - -(37) Exchange -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Arguments: hashpartitioning(ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [count(1), min(cd_dep_count#20), max(cd_dep_count#20), avg(cd_dep_count#20), min(cd_dep_employed_count#21), max(cd_dep_employed_count#21), avg(cd_dep_employed_count#21), min(cd_dep_college_count#22), max(cd_dep_college_count#22), avg(cd_dep_college_count#22)] -Aggregate Attributes [10]: [count(1)#49, min(cd_dep_count#20)#50, max(cd_dep_count#20)#51, avg(cd_dep_count#20)#52, min(cd_dep_employed_count#21)#53, max(cd_dep_employed_count#21)#54, avg(cd_dep_employed_count#21)#55, min(cd_dep_college_count#22)#56, max(cd_dep_college_count#22)#57, avg(cd_dep_college_count#22)#58] -Results [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, count(1)#49 AS cnt1#59, min(cd_dep_count#20)#50 AS min(cd_dep_count)#60, max(cd_dep_count#20)#51 AS max(cd_dep_count)#61, avg(cd_dep_count#20)#52 AS avg(cd_dep_count)#62, cd_dep_employed_count#21, count(1)#49 AS cnt2#63, min(cd_dep_employed_count#21)#53 AS min(cd_dep_employed_count)#64, max(cd_dep_employed_count#21)#54 AS max(cd_dep_employed_count)#65, avg(cd_dep_employed_count#21)#55 AS avg(cd_dep_employed_count)#66, cd_dep_college_count#22, count(1)#49 AS cnt3#67, min(cd_dep_college_count#22)#56 AS min(cd_dep_college_count)#68, max(cd_dep_college_count#22)#57 AS max(cd_dep_college_count)#69, avg(cd_dep_college_count#22)#58 AS avg(cd_dep_college_count)#70, cd_dep_count#20] - -(39) TakeOrderedAndProject -Input [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70, cd_dep_count#20] -Arguments: 100, [ca_state#16 ASC NULLS FIRST, cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_dep_count#20 ASC NULLS FIRST, cd_dep_employed_count#21 ASC NULLS FIRST, cd_dep_college_count#22 ASC NULLS FIRST], [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70] +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#23), partial_max(cd_dep_count#23), partial_avg(cd_dep_count#23), partial_min(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_avg(cd_dep_employed_count#24), partial_min(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_avg(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), min(cd_dep_count#23), max(cd_dep_count#23), avg(cd_dep_count#23), min(cd_dep_employed_count#24), max(cd_dep_employed_count#24), avg(cd_dep_employed_count#24), min(cd_dep_college_count#25), max(cd_dep_college_count#25), avg(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#23)#55, max(cd_dep_count#23)#56, avg(cd_dep_count#23)#57, min(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, avg(cd_dep_employed_count#24)#60, min(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, avg(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, count(1)#54 AS cnt1#64, min(cd_dep_count#23)#55 AS min(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#23)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#24)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#24)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, min(cd_dep_college_count#25)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#25)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#23] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#23] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/simplified.txt index 6767df5cc1..d7de10ae30 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/simplified.txt @@ -1,53 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/explain.txt index c61da07a87..a65d939cb8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/explain.txt @@ -1,46 +1,55 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- Window (27) - +- * Sort (26) - +- Exchange (25) - +- * HashAggregate (24) - +- Exchange (23) - +- * HashAggregate (22) - +- * ColumnarToRow (21) - +- CometExpand (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (31) ++- * Project (30) + +- Window (29) + +- * CometColumnarToRow (28) + +- CometSort (27) + +- CometColumnarExchange (26) + +- * HashAggregate (25) + +- * CometColumnarToRow (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6, d_year#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#6, d_year#7] @@ -63,94 +72,105 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#12] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] + +(13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11, s_state#12] - -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] - -(20) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(21) ColumnarToRow [codegen id : 1] -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(22) HashAggregate [codegen id : 1] -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((isnotnull(s_state#14) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#14, 2)) = TN)) AND isnotnull(s_store_sk#13)) + +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] + +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] + +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight + +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] + +(21) CometExpand +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] + +(22) CometHashAggregate +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#16, sum#17] -Results [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -(23) Exchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(23) CometExchange +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometColumnarToRow [codegen id : 1] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] -(24) HashAggregate [codegen id : 2] -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(25) HashAggregate [codegen id : 1] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#20, sum(UnscaledValue(ss_ext_sales_price#3))#21] -Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#13, i_class#14, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint) = 0) THEN i_category#13 END AS _w2#26] +Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#15, i_class#16, (cast((shiftright(spark_grouping_id#17, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#17, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint) = 0) THEN i_category#15 END AS _w2#26] + +(26) CometColumnarExchange +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(25) Exchange -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(27) CometSort +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26], [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST] -(26) Sort [codegen id : 3] -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST], false, 0 +(28) CometColumnarToRow [codegen id : 2] +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] -(27) Window -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] +(29) Window +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] Arguments: [rank(_w0#24) windowspecdefinition(_w1#25, _w2#26, _w0#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#27], [_w1#25, _w2#26], [_w0#24 ASC NULLS FIRST] -(28) Project [codegen id : 4] -Output [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Input [8]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] +(30) Project [codegen id : 3] +Output [5]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] +Input [8]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] -(29) TakeOrderedAndProject -Input [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] +(31) TakeOrderedAndProject +Input [5]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] +Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/extended.txt new file mode 100644 index 0000000000..d06a366e78 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/extended.txt @@ -0,0 +1,33 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: Unsupported result expressions found in: List((MakeDecimal(sum(UnscaledValue(ss_net_profit#1))#2,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#4,17,2)) AS gross_margin#5, i_category#6, i_class#7, (cast((shiftright(spark_grouping_id#8, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#8, 0) & 1) as tinyint)) AS lochierarchy#9, (MakeDecimal(sum(UnscaledValue(ss_net_profit#1))#2,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#4,17,2)) AS _w0#10, (cast((shiftright(spark_grouping_id#8, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#8, 0) & 1) as tinyint)) AS _w1#11, CASE WHEN (cast((shiftright(spark_grouping_id#8, 0) & 1) as tinyint) = 0) THEN i_category#6 END AS _w2#12)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 25 out of 29 eligible operators (86%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/simplified.txt index 255fe0d657..bb7a0cbdea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/simplified.txt @@ -1,37 +1,37 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (3) - Sort [_w1,_w2,_w0] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] - CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] - CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,spark_grouping_id,sum,sum] + CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] + CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #5 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #5 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/explain.txt index bc7fc8f71a..0e49c64c8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (26) +* CometColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) +- CometExchange (23) @@ -13,23 +13,26 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) + : : +- CometNativeScan parquet spark_catalog.default.inventory (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) +- CometProject (19) +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (17) + +- CometNativeScan parquet spark_catalog.default.catalog_sales (17) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,694,808,940]), IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] @@ -37,101 +40,111 @@ Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) A (3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#2, 16)) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#9)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] +ReadSchema: struct (5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) (6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] (7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] (8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight (9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +(10) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] +ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11, cs_sold_date_sk#12] +(17) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (18) CometFilter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) (19) CometProject -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11], [cs_item_sk#11] +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] (20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#11] -Arguments: [i_item_sk#1], [cs_item_sk#11], Inner, BuildLeft +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/extended.txt new file mode 100644 index 0000000000..92c5233469 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.inventory + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_sales + +Comet accelerated 25 out of 25 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/simplified.txt index 2a8cf3e62b..d8d49b93e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] @@ -12,17 +12,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [cs_item_sk] CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/explain.txt index dab0b8c9ad..2e0e2fc20b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/explain.txt @@ -1,44 +1,72 @@ == Physical Plan == -* ColumnarToRow (27) -+- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometBroadcastHashJoin (20) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (48) ++- CometHashAggregate (47) + +- CometExchange (46) + +- CometHashAggregate (45) + +- CometProject (44) + +- CometBroadcastHashJoin (43) + :- CometBroadcastHashJoin (30) + : :- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- CometBroadcastExchange (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - +- ReusedExchange (21) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : +- CometBroadcastExchange (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- CometBroadcastExchange (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (31) + : +- ReusedExchange (33) + +- ReusedExchange (36) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] @@ -61,83 +89,183 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(9) CometNativeScan: `spark_catalog`.`default`.`customer` +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#7, 20)) AS c_first_name#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#8, 30)) AS c_last_name#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] + +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(18) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) + +(20) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] + +(21) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(22) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] + +(23) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] + +(24) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(25) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] + +(26) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(27) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(28) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] +(29) CometBroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] + +(30) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight + +(31) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(32) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) + +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] + +(34) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(18) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] +(35) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] + +(36) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] + +(37) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight + +(38) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] + +(39) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -(19) CometBroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [c_last_name#9, c_first_name#10, d_date#11] +(40) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(20) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)], LeftSemi, BuildRight +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] -(21) ReusedExchange [Reuses operator id: 19] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] +(42) CometBroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] -(22) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#12, c_first_name#13, d_date#14] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight +(43) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight -(23) CometProject -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(44) CometProject +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(24) CometHashAggregate +(45) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -(25) CometExchange -Input [1]: [count#15] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(46) CometExchange +Input [1]: [count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(26) CometHashAggregate -Input [1]: [count#15] +(47) CometHashAggregate +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] -(27) ColumnarToRow [codegen id : 1] -Input [1]: [count(1)#16] +(48) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/extended.txt new file mode 100644 index 0000000000..ab9f89f4f0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/extended.txt @@ -0,0 +1,62 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 59 out of 59 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/simplified.txt index c2697a2c1c..b912edf4d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/simplified.txt @@ -1,7 +1,7 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [count(1),count,count(1)] + CometHashAggregate [count] [count(1),count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -15,15 +15,36 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - ReusedExchange [c_last_name,c_first_name,d_date] #5 + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/explain.txt index 3448dda988..6e9be89874 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/explain.txt @@ -1,66 +1,71 @@ == Physical Plan == -* ColumnarToRow (49) -+- CometSort (48) - +- CometColumnarExchange (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (24) - : +- * Filter (23) - : +- * HashAggregate (22) - : +- Exchange (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- BroadcastExchange (45) - +- * Project (44) - +- * Filter (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (25) - : : +- ReusedExchange (27) - : +- ReusedExchange (30) - +- CometBroadcastExchange (36) - +- CometProject (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [i_item_sk#5] @@ -79,9 +84,12 @@ Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -100,9 +108,12 @@ Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] @@ -125,139 +136,131 @@ Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -(19) ColumnarToRow [codegen id : 1] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(20) HashAggregate [codegen id : 1] +(19) CometHashAggregate Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -(21) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) HashAggregate [codegen id : 4] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] - -(23) Filter [codegen id : 4] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))))) > 1.0) END - -(24) Project [codegen id : 4] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN null ELSE (stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] - -(25) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Arguments: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(26) CometFilter -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) - -(27) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#30] - -(28) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Right output [1]: [i_item_sk#30] -Arguments: [inv_item_sk#26], [i_item_sk#30], Inner, BuildRight - -(29) CometProject -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Arguments: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30], [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] - -(30) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] - -(31) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Right output [2]: [w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_warehouse_sk#27], [w_warehouse_sk#31], Inner, BuildRight - -(32) CometProject -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32], [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] - -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_year#34, d_moy#35] - -(34) CometFilter -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(35) CometProject -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35], [d_date_sk#33, d_moy#35] - -(36) CometBroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35] - -(37) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Right output [2]: [d_date_sk#33, d_moy#35] -Arguments: [inv_date_sk#29], [d_date_sk#33], Inner, BuildRight - -(38) CometProject -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] -Arguments: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35], [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(39) ColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(40) HashAggregate [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(41) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(42) HashAggregate [codegen id : 3] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#46, avg(inv_quantity_on_hand#28)#22 AS mean#47] - -(43) Filter [codegen id : 3] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))))) > 1.0) END - -(44) Project [codegen id : 3] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN null ELSE (stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))) END AS cov#48] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] - -(45) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=3] - -(46) BroadcastHashJoin [codegen id : 4] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(47) CometColumnarExchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(48) CometSort -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST] - -(49) ColumnarToRow [codegen id : 5] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] + +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END + +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] + +(24) CometNativeScan parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#22)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) + +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] + +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight + +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] + +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] + +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight + +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] + +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN null ELSE (stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))) END AS cov#36] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/extended.txt new file mode 100644 index 0000000000..dab448e5d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/simplified.txt index d7520db711..5c8643981a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/simplified.txt @@ -1,60 +1,49 @@ -WholeStageCodegen (5) - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow InputAdapter CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [i_item_sk] #3 - CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - ReusedExchange [i_item_sk] #3 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - CometBroadcastExchange [d_date_sk,d_moy] #8 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/explain.txt index d803964b9b..368dc40f6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/explain.txt @@ -1,66 +1,71 @@ == Physical Plan == -* ColumnarToRow (49) -+- CometSort (48) - +- CometColumnarExchange (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (24) - : +- * Filter (23) - : +- * HashAggregate (22) - : +- Exchange (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- BroadcastExchange (45) - +- * Project (44) - +- * Filter (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (25) - : : +- ReusedExchange (27) - : +- ReusedExchange (30) - +- CometBroadcastExchange (36) - +- CometProject (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [i_item_sk#5] @@ -79,9 +84,12 @@ Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -100,9 +108,12 @@ Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] @@ -125,139 +136,131 @@ Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -(19) ColumnarToRow [codegen id : 1] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(20) HashAggregate [codegen id : 1] +(19) CometHashAggregate Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -(21) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) HashAggregate [codegen id : 4] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] - -(23) Filter [codegen id : 4] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))))) > 1.5) END) - -(24) Project [codegen id : 4] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN null ELSE (stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] - -(25) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Arguments: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(26) CometFilter -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) - -(27) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#30] - -(28) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Right output [1]: [i_item_sk#30] -Arguments: [inv_item_sk#26], [i_item_sk#30], Inner, BuildRight - -(29) CometProject -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Arguments: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30], [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] - -(30) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] - -(31) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Right output [2]: [w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_warehouse_sk#27], [w_warehouse_sk#31], Inner, BuildRight - -(32) CometProject -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32], [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] - -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_year#34, d_moy#35] - -(34) CometFilter -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(35) CometProject -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35], [d_date_sk#33, d_moy#35] - -(36) CometBroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35] - -(37) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Right output [2]: [d_date_sk#33, d_moy#35] -Arguments: [inv_date_sk#29], [d_date_sk#33], Inner, BuildRight - -(38) CometProject -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] -Arguments: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35], [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(39) ColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(40) HashAggregate [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(41) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(42) HashAggregate [codegen id : 3] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#46, avg(inv_quantity_on_hand#28)#22 AS mean#47] - -(43) Filter [codegen id : 3] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))))) > 1.0) END - -(44) Project [codegen id : 3] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN null ELSE (stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))) END AS cov#48] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] - -(45) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=3] - -(46) BroadcastHashJoin [codegen id : 4] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(47) CometColumnarExchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(48) CometSort -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST] - -(49) ColumnarToRow [codegen id : 5] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] + +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) + +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] + +(24) CometNativeScan parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#22)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) + +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] + +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight + +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] + +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] + +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight + +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] + +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN null ELSE (stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))) END AS cov#36] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/extended.txt new file mode 100644 index 0000000000..dab448e5d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/simplified.txt index d7520db711..5c8643981a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/simplified.txt @@ -1,60 +1,49 @@ -WholeStageCodegen (5) - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow InputAdapter CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [i_item_sk] #3 - CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - ReusedExchange [i_item_sk] #3 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - CometBroadcastExchange [d_date_sk,d_moy] #8 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/explain.txt index 06788d9a9b..4b2217ddf7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/explain.txt @@ -1,283 +1,565 @@ == Physical Plan == -TakeOrderedAndProject (51) -+- * Project (50) - +- * BroadcastHashJoin Inner BuildRight (49) - :- * Project (47) - : +- * BroadcastHashJoin Inner BuildRight (46) - : :- * Project (44) - : : +- * BroadcastHashJoin Inner BuildRight (43) - : : :- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * BroadcastHashJoin Inner BuildRight (33) - : : : : :- * Filter (17) - : : : : : +- * HashAggregate (16) - : : : : : +- Exchange (15) - : : : : : +- * HashAggregate (14) - : : : : : +- * ColumnarToRow (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : : +- BroadcastExchange (32) - : : : : +- * HashAggregate (31) - : : : : +- Exchange (30) - : : : : +- * HashAggregate (29) - : : : : +- * ColumnarToRow (28) - : : : : +- CometProject (27) - : : : : +- CometBroadcastHashJoin (26) - : : : : :- CometProject (22) - : : : : : +- CometBroadcastHashJoin (21) - : : : : : :- CometFilter (19) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (18) - : : : : : +- ReusedExchange (20) - : : : : +- CometBroadcastExchange (25) - : : : : +- CometFilter (24) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - : : : +- BroadcastExchange (37) - : : : +- * Filter (36) - : : : +- * HashAggregate (35) - : : : +- ReusedExchange (34) - : : +- BroadcastExchange (42) - : : +- * HashAggregate (41) - : : +- ReusedExchange (40) - : +- ReusedExchange (45) - +- ReusedExchange (48) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) + : : : : :- CometFilter (17) + : : : : : +- CometHashAggregate (16) + : : : : : +- CometExchange (15) + : : : : : +- CometHashAggregate (14) + : : : : : +- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometProject (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometNativeScan parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometNativeScan parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(4) CometFilter -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(9) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(11) CometBroadcastHashJoin -Left output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Right output [2]: [d_date_sk#15, d_year#16] -Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] - -(13) ColumnarToRow [codegen id : 1] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] - -(14) HashAggregate [codegen id : 1] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [2]: [sum#17, isEmpty#18] -Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] - -(15) Exchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(16) HashAggregate [codegen id : 12] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21] -Results [2]: [c_customer_id#2 AS customer_id#22, sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21 AS year_total#23] - -(17) Filter [codegen id : 12] -Input [2]: [customer_id#22, year_total#23] -Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.000000)) - -(18) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Arguments: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] + +(12) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] + +(14) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(15) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(17) CometFilter +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_customer_id#25)) - -(20) ReusedExchange [Reuses operator id: 5] -Output [6]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] - -(21) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Right output [6]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight - -(22) CometProject -Input [14]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Arguments: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37], [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(24) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(26) CometBroadcastHashJoin -Left output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Right output [2]: [d_date_sk#38, d_year#39] -Arguments: [ss_sold_date_sk#37], [d_date_sk#38], Inner, BuildRight - -(27) CometProject -Input [14]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37, d_date_sk#38, d_year#39] -Arguments: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39], [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] - -(28) ColumnarToRow [codegen id : 2] -Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] - -(29) HashAggregate [codegen id : 2] -Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [partial_sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] - -(30) Exchange -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Arguments: hashpartitioning(c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(31) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21] -Results [8]: [c_customer_id#25 AS customer_id#44, c_first_name#26 AS customer_first_name#45, c_last_name#27 AS customer_last_name#46, c_preferred_cust_flag#28 AS customer_preferred_cust_flag#47, c_birth_country#29 AS customer_birth_country#48, c_login#30 AS customer_login#49, c_email_address#31 AS customer_email_address#50, sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21 AS year_total#51] - -(32) BroadcastExchange -Input [8]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] - -(33) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#44] -Join type: Inner -Join condition: None - -(34) ReusedExchange [Reuses operator id: 15] -Output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#59, sum#60, isEmpty#61] - -(35) HashAggregate [codegen id : 5] -Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#59, sum#60, isEmpty#61] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#59] -Functions [1]: [sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#64) + cs_ext_sales_price#65) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#64) + cs_ext_sales_price#65) / 2))#66] -Results [2]: [c_customer_id#52 AS customer_id#67, sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#64) + cs_ext_sales_price#65) / 2))#66 AS year_total#68] - -(36) Filter [codegen id : 5] -Input [2]: [customer_id#67, year_total#68] -Condition : (isnotnull(year_total#68) AND (year_total#68 > 0.000000)) - -(37) BroadcastExchange -Input [2]: [customer_id#67, year_total#68] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] - -(38) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#67] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 12] -Output [11]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#68] -Input [12]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, customer_id#67, year_total#68] - -(40) ReusedExchange [Reuses operator id: 30] -Output [10]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75, d_year#76, sum#77, isEmpty#78] - -(41) HashAggregate [codegen id : 7] -Input [10]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75, d_year#76, sum#77, isEmpty#78] -Keys [8]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75, d_year#76] -Functions [1]: [sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#80) - cs_ext_discount_amt#81) + cs_ext_sales_price#82) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#80) - cs_ext_discount_amt#81) + cs_ext_sales_price#82) / 2))#66] -Results [2]: [c_customer_id#69 AS customer_id#83, sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#80) - cs_ext_discount_amt#81) + cs_ext_sales_price#82) / 2))#66 AS year_total#84] - -(42) BroadcastExchange +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)))) + +(20) CometProject +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)) AS c_customer_id#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#29, 20)) AS c_first_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#30, 30)) AS c_last_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#31, 1)) AS c_preferred_cust_flag#38, c_birth_country#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#33, 13)) AS c_login#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#34, 50)) AS c_email_address#40] + +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] + +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40] +Right output [6]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Arguments: [c_customer_sk#27], [ss_customer_sk#41], Inner, BuildRight + +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#47, d_year#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#47, d_year#48] +Condition : ((isnotnull(d_year#48) AND (d_year#48 = 2002)) AND isnotnull(d_date_sk#47)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#47, d_year#48] +Arguments: [d_date_sk#47, d_year#48] + +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Right output [2]: [d_date_sk#47, d_year#48] +Arguments: [ss_sold_date_sk#46], [d_date_sk#47], Inner, BuildRight + +(28) CometProject +Input [14]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46, d_date_sk#47, d_year#48] +Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48] + +(29) CometHashAggregate +Input [12]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48] +Keys [8]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48] +Functions [1]: [partial_sum(((((ss_ext_list_price#45 - ss_ext_wholesale_cost#44) - ss_ext_discount_amt#42) + ss_ext_sales_price#43) / 2))] + +(30) CometExchange +Input [10]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, sum#49, isEmpty#50] +Arguments: hashpartitioning(c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [10]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, sum#49, isEmpty#50] +Keys [8]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48] +Functions [1]: [sum(((((ss_ext_list_price#45 - ss_ext_wholesale_cost#44) - ss_ext_discount_amt#42) + ss_ext_sales_price#43) / 2))] + +(32) CometBroadcastExchange +Input [8]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Arguments: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Arguments: [customer_id#25], [customer_id#51], Inner, BuildRight + +(34) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(35) CometFilter +Input [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Condition : (isnotnull(c_customer_sk#59) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#60, 16)))) + +(36) CometProject +Input [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Arguments: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72], [c_customer_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#60, 16)) AS c_customer_id#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#61, 20)) AS c_first_name#68, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#62, 30)) AS c_last_name#69, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#63, 1)) AS c_preferred_cust_flag#70, c_birth_country#64, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#65, 13)) AS c_login#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#66, 50)) AS c_email_address#72] + +(37) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(38) CometFilter +Input [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Condition : isnotnull(cs_bill_customer_sk#73) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72] +Right output [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [c_customer_sk#59], [cs_bill_customer_sk#73], Inner, BuildRight + +(41) CometProject +Input [14]: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78], [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#79, d_year#80] + +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(44) CometProject +Input [14]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80], [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80] + +(45) CometHashAggregate +Input [12]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80] +Functions [1]: [partial_sum(((((cs_ext_list_price#77 - cs_ext_wholesale_cost#76) - cs_ext_discount_amt#74) + cs_ext_sales_price#75) / 2))] + +(46) CometExchange +Input [10]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, sum#81, isEmpty#82] +Arguments: hashpartitioning(c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [10]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, sum#81, isEmpty#82] +Keys [8]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80] +Functions [1]: [sum(((((cs_ext_list_price#77 - cs_ext_wholesale_cost#76) - cs_ext_discount_amt#74) + cs_ext_sales_price#75) / 2))] + +(48) CometFilter +Input [2]: [customer_id#83, year_total#84] +Condition : (isnotnull(year_total#84) AND (year_total#84 > 0.000000)) + +(49) CometBroadcastExchange Input [2]: [customer_id#83, year_total#84] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] - -(43) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#83] -Join type: Inner -Join condition: (CASE WHEN (year_total#68 > 0.000000) THEN (year_total#84 / year_total#68) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#51 / year_total#23) END) - -(44) Project [codegen id : 12] -Output [10]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84] -Input [13]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#68, customer_id#83, year_total#84] - -(45) ReusedExchange [Reuses operator id: 37] -Output [2]: [customer_id#85, year_total#86] - -(46) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#85] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 12] -Output [11]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84, year_total#86] -Input [12]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84, customer_id#85, year_total#86] - -(48) ReusedExchange [Reuses operator id: 42] -Output [2]: [customer_id#87, year_total#88] - -(49) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#87] -Join type: Inner -Join condition: (CASE WHEN (year_total#68 > 0.000000) THEN (year_total#84 / year_total#68) END > CASE WHEN (year_total#86 > 0.000000) THEN (year_total#88 / year_total#86) END) - -(50) Project [codegen id : 12] -Output [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Input [13]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84, year_total#86, customer_id#87, year_total#88] - -(51) TakeOrderedAndProject -Input [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Arguments: 100, [customer_id#44 ASC NULLS FIRST, customer_first_name#45 ASC NULLS FIRST, customer_last_name#46 ASC NULLS FIRST, customer_preferred_cust_flag#47 ASC NULLS FIRST, customer_birth_country#48 ASC NULLS FIRST, customer_login#49 ASC NULLS FIRST, customer_email_address#50 ASC NULLS FIRST], [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] +Arguments: [customer_id#83, year_total#84] + +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Right output [2]: [customer_id#83, year_total#84] +Arguments: [customer_id#25], [customer_id#83], Inner, BuildRight + +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, customer_id#83, year_total#84] +Arguments: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84], [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(53) CometFilter +Input [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Condition : (isnotnull(c_customer_sk#85) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#86, 16)))) + +(54) CometProject +Input [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Arguments: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98], [c_customer_sk#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#86, 16)) AS c_customer_id#93, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#87, 20)) AS c_first_name#94, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#88, 30)) AS c_last_name#95, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#89, 1)) AS c_preferred_cust_flag#96, c_birth_country#90, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#91, 13)) AS c_login#97, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#92, 50)) AS c_email_address#98] + +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] + +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98] +Right output [6]: [cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Arguments: [c_customer_sk#85], [cs_bill_customer_sk#99], Inner, BuildRight + +(57) CometProject +Input [14]: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Arguments: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104], [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#105, d_year#106] + +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [cs_sold_date_sk#104], [d_date_sk#105], Inner, BuildRight + +(60) CometProject +Input [14]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104, d_date_sk#105, d_year#106] +Arguments: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106], [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106] + +(61) CometHashAggregate +Input [12]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106] +Keys [8]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106] +Functions [1]: [partial_sum(((((cs_ext_list_price#103 - cs_ext_wholesale_cost#102) - cs_ext_discount_amt#100) + cs_ext_sales_price#101) / 2))] + +(62) CometExchange +Input [10]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [10]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, sum#107, isEmpty#108] +Keys [8]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106] +Functions [1]: [sum(((((cs_ext_list_price#103 - cs_ext_wholesale_cost#102) - cs_ext_discount_amt#100) + cs_ext_sales_price#101) / 2))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#109, year_total#110] +Arguments: [customer_id#109, year_total#110] + +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84] +Right output [2]: [customer_id#109, year_total#110] +Arguments: [customer_id#25], [customer_id#109], Inner, (CASE WHEN (year_total#84 > 0.000000) THEN (year_total#110 / year_total#84) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#58 / year_total#26) END), BuildRight + +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84, customer_id#109, year_total#110] +Arguments: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110], [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110] + +(67) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(68) CometFilter +Input [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Condition : (isnotnull(c_customer_sk#111) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#112, 16)))) + +(69) CometProject +Input [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Arguments: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124], [c_customer_sk#111, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#112, 16)) AS c_customer_id#119, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#113, 20)) AS c_first_name#120, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#114, 30)) AS c_last_name#121, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#115, 1)) AS c_preferred_cust_flag#122, c_birth_country#116, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#117, 13)) AS c_login#123, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#118, 50)) AS c_email_address#124] + +(70) CometNativeScan parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#130)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(71) CometFilter +Input [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Condition : isnotnull(ws_bill_customer_sk#125) + +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] + +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124] +Right output [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [c_customer_sk#111], [ws_bill_customer_sk#125], Inner, BuildRight + +(74) CometProject +Input [14]: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130], [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] + +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#131, d_year#132] + +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Right output [2]: [d_date_sk#131, d_year#132] +Arguments: [ws_sold_date_sk#130], [d_date_sk#131], Inner, BuildRight + +(77) CometProject +Input [14]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130, d_date_sk#131, d_year#132] +Arguments: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132], [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132] + +(78) CometHashAggregate +Input [12]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132] +Keys [8]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132] +Functions [1]: [partial_sum(((((ws_ext_list_price#129 - ws_ext_wholesale_cost#128) - ws_ext_discount_amt#126) + ws_ext_sales_price#127) / 2))] + +(79) CometExchange +Input [10]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, sum#133, isEmpty#134] +Arguments: hashpartitioning(c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(80) CometHashAggregate +Input [10]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, sum#133, isEmpty#134] +Keys [8]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132] +Functions [1]: [sum(((((ws_ext_list_price#129 - ws_ext_wholesale_cost#128) - ws_ext_discount_amt#126) + ws_ext_sales_price#127) / 2))] + +(81) CometFilter +Input [2]: [customer_id#135, year_total#136] +Condition : (isnotnull(year_total#136) AND (year_total#136 > 0.000000)) + +(82) CometBroadcastExchange +Input [2]: [customer_id#135, year_total#136] +Arguments: [customer_id#135, year_total#136] + +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110] +Right output [2]: [customer_id#135, year_total#136] +Arguments: [customer_id#25], [customer_id#135], Inner, BuildRight + +(84) CometProject +Input [12]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, customer_id#135, year_total#136] +Arguments: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136], [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136] + +(85) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(86) CometFilter +Input [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Condition : (isnotnull(c_customer_sk#137) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#138, 16)))) + +(87) CometProject +Input [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Arguments: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150], [c_customer_sk#137, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#138, 16)) AS c_customer_id#145, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#139, 20)) AS c_first_name#146, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#140, 30)) AS c_last_name#147, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#141, 1)) AS c_preferred_cust_flag#148, c_birth_country#142, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#143, 13)) AS c_login#149, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#144, 50)) AS c_email_address#150] + +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] + +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150] +Right output [6]: [ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Arguments: [c_customer_sk#137], [ws_bill_customer_sk#151], Inner, BuildRight + +(90) CometProject +Input [14]: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Arguments: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156], [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] + +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#157, d_year#158] + +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Right output [2]: [d_date_sk#157, d_year#158] +Arguments: [ws_sold_date_sk#156], [d_date_sk#157], Inner, BuildRight + +(93) CometProject +Input [14]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156, d_date_sk#157, d_year#158] +Arguments: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158], [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158] + +(94) CometHashAggregate +Input [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158] +Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158] +Functions [1]: [partial_sum(((((ws_ext_list_price#155 - ws_ext_wholesale_cost#154) - ws_ext_discount_amt#152) + ws_ext_sales_price#153) / 2))] + +(95) CometExchange +Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, sum#159, isEmpty#160] +Arguments: hashpartitioning(c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(96) CometHashAggregate +Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, sum#159, isEmpty#160] +Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158] +Functions [1]: [sum(((((ws_ext_list_price#155 - ws_ext_wholesale_cost#154) - ws_ext_discount_amt#152) + ws_ext_sales_price#153) / 2))] + +(97) CometBroadcastExchange +Input [2]: [customer_id#161, year_total#162] +Arguments: [customer_id#161, year_total#162] + +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136] +Right output [2]: [customer_id#161, year_total#162] +Arguments: [customer_id#25], [customer_id#161], Inner, (CASE WHEN (year_total#84 > 0.000000) THEN (year_total#110 / year_total#84) END > CASE WHEN (year_total#136 > 0.000000) THEN (year_total#162 / year_total#136) END), BuildRight + +(99) CometProject +Input [13]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136, customer_id#161, year_total#162] +Arguments: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57], [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] + +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#51 ASC NULLS FIRST,customer_first_name#52 ASC NULLS FIRST,customer_last_name#53 ASC NULLS FIRST,customer_preferred_cust_flag#54 ASC NULLS FIRST,customer_birth_country#55 ASC NULLS FIRST,customer_login#56 ASC NULLS FIRST,customer_email_address#57 ASC NULLS FIRST], output=[customer_id#51,customer_first_name#52,customer_last_name#53,customer_preferred_cust_flag#54,customer_birth_country#55,customer_login#56,customer_email_address#57]), [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57], 100, 0, [customer_id#51 ASC NULLS FIRST, customer_first_name#52 ASC NULLS FIRST, customer_last_name#53 ASC NULLS FIRST, customer_preferred_cust_flag#54 ASC NULLS FIRST, customer_birth_country#55 ASC NULLS FIRST, customer_login#56 ASC NULLS FIRST, customer_email_address#57 ASC NULLS FIRST], [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] + +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/extended.txt new file mode 100644 index 0000000000..7b0b7d1cc0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/extended.txt @@ -0,0 +1,117 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 114 out of 114 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/simplified.txt index 12ceb852df..f837111fe3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/simplified.txt @@ -1,68 +1,103 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - WholeStageCodegen (12) - Project [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - WholeStageCodegen (1) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #1 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #5 - InputAdapter - ReusedExchange [customer_id,year_total] #7 - InputAdapter - ReusedExchange [customer_id,year_total] #8 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 + CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/explain.txt index 1cad180942..5ff8fb2c60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/explain.txt @@ -1,41 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) : : :- CometProject (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (5) - : : +- CometBroadcastExchange (14) - : : +- CometFilter (13) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (12) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - +- CometBroadcastExchange (25) - +- CometFilter (24) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (26) + +- CometFilter (25) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] @@ -49,9 +54,12 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(5) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct (6) CometFilter Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] @@ -78,95 +86,104 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -(12) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(12) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (13) CometFilter Input [2]: [w_warehouse_sk#10, w_state#11] Condition : isnotnull(w_warehouse_sk#10) -(14) CometBroadcastExchange +(14) CometProject Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#11, 2)) AS w_state#12] -(15) CometBroadcastHashJoin +(15) CometBroadcastExchange +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] + +(16) CometBroadcastHashJoin Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -Right output [2]: [w_warehouse_sk#10, w_state#11] +Right output [2]: [w_warehouse_sk#10, w_state#12] Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight -(16) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(18) CometFilter -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) - -(19) CometProject -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13], [i_item_sk#12, i_item_id#13] - -(20) CometBroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: [i_item_sk#12, i_item_id#13] - -(21) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Right output [2]: [i_item_sk#12, i_item_id#13] -Arguments: [cs_item_sk#2], [i_item_sk#12], Inner, BuildRight - -(22) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(24) CometFilter -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(26) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#5], [d_date_sk#15], Inner, BuildRight - -(27) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] -Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16], [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] - -(28) ColumnarToRow [codegen id : 1] -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] - -(29) HashAggregate [codegen id : 1] -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [4]: [sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] - -(30) Exchange -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(31) HashAggregate [codegen id : 2] -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26] -Results [4]: [w_state#11, i_item_id#13, sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25 AS sales_before#27, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26 AS sales_after#28] - -(32) TakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#27, sales_after#28] -Arguments: 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#27, sales_after#28] +(17) CometProject +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] + +(18) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] +ReadSchema: struct + +(19) CometFilter +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) + +(20) CometProject +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#14, 16)) AS i_item_id#16] + +(21) CometBroadcastExchange +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] + +(22) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight + +(23) CometProject +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#17, d_date#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] + +(27) CometBroadcastHashJoin +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight + +(28) CometProject +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] + +(29) CometHashAggregate +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(30) CometExchange +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(31) CometHashAggregate +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(32) CometTakeOrderedAndProject +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] + +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/extended.txt new file mode 100644 index 0000000000..7a22197a97 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/simplified.txt index 3d73ff4008..98932cc775 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/simplified.txt @@ -1,36 +1,35 @@ -TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - WholeStageCodegen (2) - HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_state,i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] - CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] - CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] - CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] - CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #2 - CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometExchange [cr_order_number,cr_item_sk] #3 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_state] #4 - CometFilter [w_warehouse_sk,w_state] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_state] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [w_state,i_item_id,sales_before,sales_after,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] + CometExchange [w_state,i_item_id] #1 + CometHashAggregate [d_date,cs_sales_price,cr_refunded_cash] [w_state,i_item_id,sum,isEmpty,sum,isEmpty] + CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] + CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] + CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] + CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] + CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] + CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #2 + CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] + CometExchange [cr_order_number,cr_item_sk] #3 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_state] #4 + CometProject [w_state] [w_warehouse_sk,w_state] + CometFilter [w_warehouse_sk,w_state] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/explain.txt index e1a62cf025..786423a455 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : +- CometNativeScan parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,12 +17,15 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometNativeScan: `spark_catalog`.`default`.`item` (4) + +- CometNativeScan parquet spark_catalog.default.item (4) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Arguments: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)] +ReadSchema: struct (2) CometFilter Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] @@ -32,71 +35,73 @@ Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_m Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometNativeScan: `spark_catalog`.`default`.`item` +(4) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct (5) CometFilter Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) +Condition : (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Women ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = powder ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = khaki )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Ounce ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Oz ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = brown ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = honeydew )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Bunch ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Ton ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = N/A ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = small ))))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Men ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = floral ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = deep )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = N/A ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Dozen ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = petite ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = light ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = cornflower )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Box ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Pound ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large )))))) OR (((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Women ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = midnight ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = snow )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Pallet ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Gross ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = cyan ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = papaya )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Cup ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Dram ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = N/A ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = small ))))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Men ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = orange ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = frosted )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Each ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Tbl ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = petite ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = forest ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = ghost )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Lb ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Bundle ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large ))))))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#5, 50)))) (6) CometProject Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_manufact#5], [i_manufact#5] +Arguments: [i_manufact#9], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#5, 50)) AS i_manufact#9] (7) CometHashAggregate -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] +Input [1]: [i_manufact#9] +Keys [1]: [i_manufact#9] Functions [1]: [partial_count(1)] (8) CometExchange -Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_manufact#9, count#10] +Arguments: hashpartitioning(i_manufact#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (9) CometHashAggregate -Input [2]: [i_manufact#5, count#9] -Keys [1]: [i_manufact#5] +Input [2]: [i_manufact#9, count#10] +Keys [1]: [i_manufact#9] Functions [1]: [count(1)] (10) CometFilter -Input [2]: [item_cnt#10, i_manufact#5] -Condition : (item_cnt#10 > 0) +Input [2]: [item_cnt#11, i_manufact#9] +Condition : (item_cnt#11 > 0) (11) CometProject -Input [2]: [item_cnt#10, i_manufact#5] -Arguments: [i_manufact#5], [i_manufact#5] +Input [2]: [item_cnt#11, i_manufact#9] +Arguments: [i_manufact#9], [i_manufact#9] (12) CometBroadcastExchange -Input [1]: [i_manufact#5] -Arguments: [i_manufact#5] +Input [1]: [i_manufact#9] +Arguments: [i_manufact#9] (13) CometBroadcastHashJoin Left output [2]: [i_manufact#2, i_product_name#3] -Right output [1]: [i_manufact#5] -Arguments: [i_manufact#2], [i_manufact#5], Inner, BuildRight +Right output [1]: [i_manufact#9] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#2, 50))], [i_manufact#9], Inner, BuildRight (14) CometProject -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] -Arguments: [i_product_name#3], [i_product_name#3] +Input [3]: [i_manufact#2, i_product_name#3, i_manufact#9] +Arguments: [i_product_name#12], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#3, 50)) AS i_product_name#12] (15) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] Functions: [] (16) CometExchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [1]: [i_product_name#12] +Arguments: hashpartitioning(i_product_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (17) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] Functions: [] (18) CometTakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#3 ASC NULLS FIRST], output=[i_product_name#3]), [i_product_name#3], 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] +Input [1]: [i_product_name#12] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#12 ASC NULLS FIRST], output=[i_product_name#12]), [i_product_name#12], 100, 0, [i_product_name#12 ASC NULLS FIRST], [i_product_name#12] -(19) ColumnarToRow [codegen id : 1] -Input [1]: [i_product_name#3] +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [i_product_name#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/extended.txt new file mode 100644 index 0000000000..b40ebddb6f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/simplified.txt index ea03f38e2c..2557f51684 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_product_name] CometHashAggregate [i_product_name] CometExchange [i_product_name] #1 CometHashAggregate [i_product_name] - CometProject [i_product_name] + CometProject [i_product_name] [i_product_name] CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_manufact_id,i_manufact,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] - CometHashAggregate [item_cnt,i_manufact,count,count(1)] + CometHashAggregate [count] [item_cnt,i_manufact,count(1)] CometExchange [i_manufact] #3 CometHashAggregate [i_manufact,count] - CometProject [i_manufact] + CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact,i_size,i_color,i_units] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/explain.txt index 63c16c9fbf..a99b1a1164 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/explain.txt @@ -1,28 +1,31 @@ == Physical Plan == TakeOrderedAndProject (19) +- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] @@ -63,43 +73,41 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnul (11) CometProject Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9], [i_item_sk#7, i_category_id#8, i_category#9] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11], [i_item_sk#7, i_category_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9] +Input [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_category_id#8, i_category#9] +Right output [3]: [i_item_sk#7, i_category_id#8, i_category#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] -(16) HashAggregate [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -(17) Exchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] -(18) HashAggregate [codegen id : 2] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +(18) HashAggregate [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_category_id#8, i_category#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] +Results [4]: [d_year#2, i_category_id#8, i_category#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] (19) TakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] -Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#14] +Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#11 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/extended.txt new file mode 100644 index 0000000000..22e54569bc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/extended.txt @@ -0,0 +1,21 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#1, i_category_id#2, i_category#3, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#4))#5,17,2) AS sum(ss_ext_sales_price)#6)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 16 out of 18 eligible operators (88%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/simplified.txt index 001869f2a7..e655ac6b9c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/simplified.txt @@ -1,23 +1,21 @@ TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum] - InputAdapter - Exchange [d_year,i_category_id,i_category] #1 - WholeStageCodegen (1) - HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 - CometProject [i_item_sk,i_category_id,i_category] - CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category_id,i_category,i_manager_id] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_category_id,i_category] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_category_id,i_category,sum] + CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 + CometProject [i_category] [i_item_sk,i_category_id,i_category] + CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/explain.txt index 1c898bf879..6e47ca0462 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/explain.txt @@ -1,28 +1,31 @@ == Physical Plan == TakeOrderedAndProject (19) +- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + +- CometNativeScan parquet spark_catalog.default.store (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_year#2, d_day_name#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] @@ -30,76 +33,81 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk (3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_day_name#3], [d_date_sk#1, d_day_name#3] +Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(d_day_name#3, 9)) AS d_day_name#4] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_store_sk#5) (6) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] (7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_day_name#3] -Right output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight +Left output [2]: [d_date_sk#1, d_day_name#4] +Right output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight (8) CometProject -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_day_name#3, ss_store_sk#4, ss_sales_price#5], [d_day_name#3, ss_store_sk#4, ss_sales_price#5] +Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] +(9) CometNativeScan parquet spark_catalog.default.store +Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8)) (11) CometProject -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9], [s_store_sk#7, s_store_id#8, s_store_name#9] +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10], [s_store_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#9, 16)) AS s_store_id#12, s_store_name#10] (12) CometBroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9] +Input [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10] (13) CometBroadcastHashJoin -Left output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Right output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [ss_store_sk#4], [s_store_sk#7], Inner, BuildRight +Left output [3]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6] +Right output [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [ss_store_sk#5], [s_store_sk#8], Inner, BuildRight (14) CometProject -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9], [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] - -(16) HashAggregate [codegen id : 1] -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Results [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] - -(17) Exchange -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(18) HashAggregate [codegen id : 2] -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31] -Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] +Input [6]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6, s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10], [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] + +(15) CometHashAggregate +Input [4]: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] + +(16) CometExchange +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(s_store_name#10, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] + +(18) HashAggregate [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))#26] +Results [9]: [s_store_name#10, s_store_id#12, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END))#20,17,2) AS sun_sales#27, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END))#21,17,2) AS mon_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END))#22,17,2) AS tue_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END))#23,17,2) AS wed_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END))#24,17,2) AS thu_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END))#25,17,2) AS fri_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))#26,17,2) AS sat_sales#33] (19) TakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] -Arguments: 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#32 ASC NULLS FIRST, mon_sales#33 ASC NULLS FIRST, tue_sales#34 ASC NULLS FIRST, wed_sales#35 ASC NULLS FIRST, thu_sales#36 ASC NULLS FIRST, fri_sales#37 ASC NULLS FIRST, sat_sales#38 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#27, mon_sales#28, tue_sales#29, wed_sales#30, thu_sales#31, fri_sales#32, sat_sales#33] +Arguments: 100, [s_store_name#10 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sun_sales#27 ASC NULLS FIRST, mon_sales#28 ASC NULLS FIRST, tue_sales#29 ASC NULLS FIRST, wed_sales#30 ASC NULLS FIRST, thu_sales#31 ASC NULLS FIRST, fri_sales#32 ASC NULLS FIRST, sat_sales#33 ASC NULLS FIRST], [s_store_name#10, s_store_id#12, sun_sales#27, mon_sales#28, tue_sales#29, wed_sales#30, thu_sales#31, fri_sales#32, sat_sales#33] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/extended.txt new file mode 100644 index 0000000000..514f2028ba --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/extended.txt @@ -0,0 +1,21 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_name#1, s_store_id#2, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#4 END))#5,17,2) AS sun_sales#6, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#4 END))#7,17,2) AS mon_sales#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#4 END))#9,17,2) AS tue_sales#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#4 END))#11,17,2) AS wed_sales#12, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#4 END))#13,17,2) AS thu_sales#14, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#4 END))#15,17,2) AS fri_sales#16, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#4 END))#17,17,2) AS sat_sales#18)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 16 out of 18 eligible operators (88%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/simplified.txt index 151a17bda1..eccd118cf8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/simplified.txt @@ -1,23 +1,21 @@ TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_store_id] #1 - WholeStageCodegen (1) - HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] - CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] - CometProject [d_day_name,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_day_name] - CometFilter [d_date_sk,d_year,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_day_name] - CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometProject [s_store_sk,s_store_id,s_store_name] - CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometColumnarToRow + InputAdapter + CometExchange [s_store_name,s_store_id] #1 + CometHashAggregate [d_day_name,ss_sales_price] [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] + CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] + CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] + CometProject [d_day_name,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [d_day_name] [d_date_sk,d_day_name] + CometFilter [d_date_sk,d_year,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/explain.txt index 8c3051bc5d..835c6f13f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/explain.txt @@ -1,52 +1,59 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Project (33) - : : +- * SortMergeJoin Inner (32) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * Filter (15) - : : : +- Window (14) - : : : +- WindowGroupLimit (13) - : : : +- * Sort (12) - : : : +- Exchange (11) - : : : +- WindowGroupLimit (10) - : : : +- * Sort (9) - : : : +- * Filter (8) - : : : +- * HashAggregate (7) - : : : +- Exchange (6) - : : : +- * HashAggregate (5) - : : : +- * ColumnarToRow (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- * Sort (31) - : : +- Exchange (30) - : : +- * Project (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- WindowGroupLimit (26) - : : +- * Sort (25) - : : +- Exchange (24) - : : +- WindowGroupLimit (23) - : : +- * Sort (22) - : : +- * Filter (21) - : : +- * HashAggregate (20) - : : +- ReusedExchange (19) - : +- BroadcastExchange (37) - : +- * ColumnarToRow (36) - : +- CometFilter (35) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (34) - +- ReusedExchange (40) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometProject (45) + +- CometBroadcastHashJoin (44) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometSortMergeJoin (35) + : : :- CometSort (19) + : : : +- CometColumnarExchange (18) + : : : +- * Project (17) + : : : +- * Filter (16) + : : : +- Window (15) + : : : +- WindowGroupLimit (14) + : : : +- * CometColumnarToRow (13) + : : : +- CometSort (12) + : : : +- CometColumnarExchange (11) + : : : +- WindowGroupLimit (10) + : : : +- * CometColumnarToRow (9) + : : : +- CometSort (8) + : : : +- CometFilter (7) + : : : +- CometHashAggregate (6) + : : : +- CometExchange (5) + : : : +- CometHashAggregate (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometSort (34) + : : +- CometColumnarExchange (33) + : : +- * Project (32) + : : +- * Filter (31) + : : +- Window (30) + : : +- WindowGroupLimit (29) + : : +- * CometColumnarToRow (28) + : : +- CometSort (27) + : : +- CometColumnarExchange (26) + : : +- WindowGroupLimit (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometFilter (22) + : : +- CometHashAggregate (21) + : : +- ReusedExchange (20) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometNativeScan parquet spark_catalog.default.item (37) + +- ReusedExchange (43) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] @@ -56,222 +63,228 @@ Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [ss_item_sk#1, ss_net_profit#3], [ss_item_sk#1, ss_net_profit#3] -(4) ColumnarToRow [codegen id : 1] -Input [2]: [ss_item_sk#1, ss_net_profit#3] - -(5) HashAggregate [codegen id : 1] +(4) CometHashAggregate Input [2]: [ss_item_sk#1, ss_net_profit#3] Keys [1]: [ss_item_sk#1] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#5, count#6] -Results [3]: [ss_item_sk#1, sum#7, count#8] -(6) Exchange -Input [3]: [ss_item_sk#1, sum#7, count#8] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(5) CometExchange +Input [3]: [ss_item_sk#1, sum#5, count#6] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(7) HashAggregate [codegen id : 2] -Input [3]: [ss_item_sk#1, sum#7, count#8] +(6) CometHashAggregate +Input [3]: [ss_item_sk#1, sum#5, count#6] Keys [1]: [ss_item_sk#1] Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#9] -Results [2]: [ss_item_sk#1 AS item_sk#10, cast((avg(UnscaledValue(ss_net_profit#3))#9 / 100.0) as decimal(11,6)) AS rank_col#11] -(8) Filter [codegen id : 2] -Input [2]: [item_sk#10, rank_col#11] -Condition : (isnotnull(rank_col#11) AND (cast(rank_col#11 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#12, [id=#13]))) +(7) CometFilter +Input [2]: [item_sk#7, rank_col#8] +Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#9, [id=#10]))) -(9) Sort [codegen id : 2] -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 +(8) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(9) CometColumnarToRow [codegen id : 1] +Input [2]: [item_sk#7, rank_col#8] (10) WindowGroupLimit -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], rank(rank_col#11), 10, Partial +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Partial + +(11) CometColumnarExchange +Input [2]: [item_sk#7, rank_col#8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(12) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(13) CometColumnarToRow [codegen id : 2] +Input [2]: [item_sk#7, rank_col#8] + +(14) WindowGroupLimit +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Final + +(15) Window +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] + +(16) Filter [codegen id : 3] +Input [3]: [item_sk#7, rank_col#8, rnk#11] +Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) + +(17) Project [codegen id : 3] +Output [2]: [item_sk#7, rnk#11] +Input [3]: [item_sk#7, rank_col#8, rnk#11] + +(18) CometColumnarExchange +Input [2]: [item_sk#7, rnk#11] +Arguments: hashpartitioning(rnk#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(19) CometSort +Input [2]: [item_sk#7, rnk#11] +Arguments: [item_sk#7, rnk#11], [rnk#11 ASC NULLS FIRST] + +(20) ReusedExchange [Reuses operator id: 5] +Output [3]: [ss_item_sk#12, sum#13, count#14] + +(21) CometHashAggregate +Input [3]: [ss_item_sk#12, sum#13, count#14] +Keys [1]: [ss_item_sk#12] +Functions [1]: [avg(UnscaledValue(ss_net_profit#15))] + +(22) CometFilter +Input [2]: [item_sk#16, rank_col#17] +Condition : (isnotnull(rank_col#17) AND (cast(rank_col#17 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) + +(23) CometSort +Input [2]: [item_sk#16, rank_col#17] +Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] + +(24) CometColumnarToRow [codegen id : 4] +Input [2]: [item_sk#16, rank_col#17] + +(25) WindowGroupLimit +Input [2]: [item_sk#16, rank_col#17] +Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Partial + +(26) CometColumnarExchange +Input [2]: [item_sk#16, rank_col#17] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(27) CometSort +Input [2]: [item_sk#16, rank_col#17] +Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] + +(28) CometColumnarToRow [codegen id : 5] +Input [2]: [item_sk#16, rank_col#17] + +(29) WindowGroupLimit +Input [2]: [item_sk#16, rank_col#17] +Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Final + +(30) Window +Input [2]: [item_sk#16, rank_col#17] +Arguments: [rank(rank_col#17) windowspecdefinition(rank_col#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#18], [rank_col#17 DESC NULLS LAST] + +(31) Filter [codegen id : 6] +Input [3]: [item_sk#16, rank_col#17, rnk#18] +Condition : ((rnk#18 < 11) AND isnotnull(item_sk#16)) + +(32) Project [codegen id : 6] +Output [2]: [item_sk#16, rnk#18] +Input [3]: [item_sk#16, rank_col#17, rnk#18] + +(33) CometColumnarExchange +Input [2]: [item_sk#16, rnk#18] +Arguments: hashpartitioning(rnk#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(34) CometSort +Input [2]: [item_sk#16, rnk#18] +Arguments: [item_sk#16, rnk#18], [rnk#18 ASC NULLS FIRST] + +(35) CometSortMergeJoin +Left output [2]: [item_sk#7, rnk#11] +Right output [2]: [item_sk#16, rnk#18] +Arguments: [rnk#11], [rnk#18], Inner + +(36) CometProject +Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] +Arguments: [item_sk#7, rnk#11, item_sk#16], [item_sk#7, rnk#11, item_sk#16] + +(37) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#19, i_product_name#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(38) CometFilter +Input [2]: [i_item_sk#19, i_product_name#20] +Condition : isnotnull(i_item_sk#19) + +(39) CometProject +Input [2]: [i_item_sk#19, i_product_name#20] +Arguments: [i_item_sk#19, i_product_name#21], [i_item_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#20, 50)) AS i_product_name#21] -(11) Exchange -Input [2]: [item_sk#10, rank_col#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(40) CometBroadcastExchange +Input [2]: [i_item_sk#19, i_product_name#21] +Arguments: [i_item_sk#19, i_product_name#21] -(12) Sort [codegen id : 3] -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 +(41) CometBroadcastHashJoin +Left output [3]: [item_sk#7, rnk#11, item_sk#16] +Right output [2]: [i_item_sk#19, i_product_name#21] +Arguments: [item_sk#7], [i_item_sk#19], Inner, BuildRight -(13) WindowGroupLimit -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], rank(rank_col#11), 10, Final +(42) CometProject +Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#21] +Arguments: [rnk#11, item_sk#16, i_product_name#21], [rnk#11, item_sk#16, i_product_name#21] -(14) Window -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank(rank_col#11) windowspecdefinition(rank_col#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#11 ASC NULLS FIRST] - -(15) Filter [codegen id : 4] -Input [3]: [item_sk#10, rank_col#11, rnk#14] -Condition : ((rnk#14 < 11) AND isnotnull(item_sk#10)) - -(16) Project [codegen id : 4] -Output [2]: [item_sk#10, rnk#14] -Input [3]: [item_sk#10, rank_col#11, rnk#14] +(43) ReusedExchange [Reuses operator id: 40] +Output [2]: [i_item_sk#22, i_product_name#23] -(17) Exchange -Input [2]: [item_sk#10, rnk#14] -Arguments: hashpartitioning(rnk#14, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 5] -Input [2]: [item_sk#10, rnk#14] -Arguments: [rnk#14 ASC NULLS FIRST], false, 0 - -(19) ReusedExchange [Reuses operator id: 6] -Output [3]: [ss_item_sk#15, sum#16, count#17] - -(20) HashAggregate [codegen id : 7] -Input [3]: [ss_item_sk#15, sum#16, count#17] -Keys [1]: [ss_item_sk#15] -Functions [1]: [avg(UnscaledValue(ss_net_profit#18))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#18))#19] -Results [2]: [ss_item_sk#15 AS item_sk#20, cast((avg(UnscaledValue(ss_net_profit#18))#19 / 100.0) as decimal(11,6)) AS rank_col#21] - -(21) Filter [codegen id : 7] -Input [2]: [item_sk#20, rank_col#21] -Condition : (isnotnull(rank_col#21) AND (cast(rank_col#21 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#12, [id=#13]))) - -(22) Sort [codegen id : 7] -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], false, 0 - -(23) WindowGroupLimit -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], rank(rank_col#21), 10, Partial - -(24) Exchange -Input [2]: [item_sk#20, rank_col#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(25) Sort [codegen id : 8] -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], false, 0 - -(26) WindowGroupLimit -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], rank(rank_col#21), 10, Final - -(27) Window -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank(rank_col#21) windowspecdefinition(rank_col#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#22], [rank_col#21 DESC NULLS LAST] - -(28) Filter [codegen id : 9] -Input [3]: [item_sk#20, rank_col#21, rnk#22] -Condition : ((rnk#22 < 11) AND isnotnull(item_sk#20)) - -(29) Project [codegen id : 9] -Output [2]: [item_sk#20, rnk#22] -Input [3]: [item_sk#20, rank_col#21, rnk#22] - -(30) Exchange -Input [2]: [item_sk#20, rnk#22] -Arguments: hashpartitioning(rnk#22, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) Sort [codegen id : 10] -Input [2]: [item_sk#20, rnk#22] -Arguments: [rnk#22 ASC NULLS FIRST], false, 0 - -(32) SortMergeJoin [codegen id : 13] -Left keys [1]: [rnk#14] -Right keys [1]: [rnk#22] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 13] -Output [3]: [item_sk#10, rnk#14, item_sk#20] -Input [4]: [item_sk#10, rnk#14, item_sk#20, rnk#22] - -(34) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#23, i_product_name#24] -Arguments: [i_item_sk#23, i_product_name#24] +(44) CometBroadcastHashJoin +Left output [3]: [rnk#11, item_sk#16, i_product_name#21] +Right output [2]: [i_item_sk#22, i_product_name#23] +Arguments: [item_sk#16], [i_item_sk#22], Inner, BuildRight -(35) CometFilter -Input [2]: [i_item_sk#23, i_product_name#24] -Condition : isnotnull(i_item_sk#23) +(45) CometProject +Input [5]: [rnk#11, item_sk#16, i_product_name#21, i_item_sk#22, i_product_name#23] +Arguments: [rnk#11, best_performing#24, worst_performing#25], [rnk#11, i_product_name#21 AS best_performing#24, i_product_name#23 AS worst_performing#25] -(36) ColumnarToRow [codegen id : 11] -Input [2]: [i_item_sk#23, i_product_name#24] - -(37) BroadcastExchange -Input [2]: [i_item_sk#23, i_product_name#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [item_sk#10] -Right keys [1]: [i_item_sk#23] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 13] -Output [3]: [rnk#14, item_sk#20, i_product_name#24] -Input [5]: [item_sk#10, rnk#14, item_sk#20, i_item_sk#23, i_product_name#24] - -(40) ReusedExchange [Reuses operator id: 37] -Output [2]: [i_item_sk#25, i_product_name#26] +(46) CometTakeOrderedAndProject +Input [3]: [rnk#11, best_performing#24, worst_performing#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[rnk#11 ASC NULLS FIRST], output=[rnk#11,best_performing#24,worst_performing#25]), [rnk#11, best_performing#24, worst_performing#25], 100, 0, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#24, worst_performing#25] -(41) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [item_sk#20] -Right keys [1]: [i_item_sk#25] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 13] -Output [3]: [rnk#14, i_product_name#24 AS best_performing#27, i_product_name#26 AS worst_performing#28] -Input [5]: [rnk#14, item_sk#20, i_product_name#24, i_item_sk#25, i_product_name#26] - -(43) TakeOrderedAndProject -Input [3]: [rnk#14, best_performing#27, worst_performing#28] -Arguments: 100, [rnk#14 ASC NULLS FIRST], [rnk#14, best_performing#27, worst_performing#28] +(47) CometColumnarToRow [codegen id : 7] +Input [3]: [rnk#11, best_performing#24, worst_performing#25] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * ColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (44) - - -(44) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] -Arguments: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] - -(45) CometFilter -Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] -Condition : ((isnotnull(ss_store_sk#30) AND (ss_store_sk#30 = 4)) AND isnull(ss_addr_sk#29)) - -(46) CometProject -Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] -Arguments: [ss_store_sk#30, ss_net_profit#31], [ss_store_sk#30, ss_net_profit#31] - -(47) ColumnarToRow [codegen id : 1] -Input [2]: [ss_store_sk#30, ss_net_profit#31] - -(48) HashAggregate [codegen id : 1] -Input [2]: [ss_store_sk#30, ss_net_profit#31] -Keys [1]: [ss_store_sk#30] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#31))] -Aggregate Attributes [2]: [sum#33, count#34] -Results [3]: [ss_store_sk#30, sum#35, count#36] - -(49) Exchange -Input [3]: [ss_store_sk#30, sum#35, count#36] -Arguments: hashpartitioning(ss_store_sk#30, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(50) HashAggregate [codegen id : 2] -Input [3]: [ss_store_sk#30, sum#35, count#36] -Keys [1]: [ss_store_sk#30] -Functions [1]: [avg(UnscaledValue(ss_net_profit#31))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#31))#37] -Results [1]: [cast((avg(UnscaledValue(ss_net_profit#31))#37 / 100.0) as decimal(11,6)) AS rank_col#38] - -Subquery:2 Hosting operator id = 21 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] +* CometColumnarToRow (54) ++- CometHashAggregate (53) + +- CometExchange (52) + +- CometHashAggregate (51) + +- CometProject (50) + +- CometFilter (49) + +- CometNativeScan parquet spark_catalog.default.store_sales (48) + + +(48) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] +ReadSchema: struct + +(49) CometFilter +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_store_sk#27) AND (ss_store_sk#27 = 4)) AND isnull(ss_addr_sk#26)) + +(50) CometProject +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Arguments: [ss_store_sk#27, ss_net_profit#28], [ss_store_sk#27, ss_net_profit#28] + +(51) CometHashAggregate +Input [2]: [ss_store_sk#27, ss_net_profit#28] +Keys [1]: [ss_store_sk#27] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#28))] + +(52) CometExchange +Input [3]: [ss_store_sk#27, sum#30, count#31] +Arguments: hashpartitioning(ss_store_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(53) CometHashAggregate +Input [3]: [ss_store_sk#27, sum#30, count#31] +Keys [1]: [ss_store_sk#27] +Functions [1]: [avg(UnscaledValue(ss_net_profit#28))] + +(54) CometColumnarToRow [codegen id : 1] +Input [1]: [rank_col#32] + +Subquery:2 Hosting operator id = 22 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/extended.txt new file mode 100644 index 0000000000..6595583fa5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/extended.txt @@ -0,0 +1,65 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometColumnarExchange + : : : +- Project + : : : +- Filter + : : : +- Window + : : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometColumnarExchange + : : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometFilter + : : : : +- Subquery + : : : : +- CometColumnarToRow + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometSort + : : +- CometColumnarExchange + : : +- Project + : : +- Filter + : : +- Window + : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 45 out of 57 eligible operators (78%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/simplified.txt index de76347fe1..02d8749188 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/simplified.txt @@ -1,81 +1,72 @@ -TakeOrderedAndProject [rnk,best_performing,worst_performing] - WholeStageCodegen (13) - Project [rnk,i_product_name,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [rnk,item_sk,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [item_sk,rnk,item_sk] - SortMergeJoin [rnk,rnk] - InputAdapter - WholeStageCodegen (5) - Sort [rnk] - InputAdapter - Exchange [rnk] #1 - WholeStageCodegen (4) - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WindowGroupLimit [rank_col] - WholeStageCodegen (3) - Sort [rank_col] - InputAdapter - Exchange #2 +WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [rnk,best_performing,worst_performing] + CometProject [i_product_name,i_product_name] [rnk,best_performing,worst_performing] + CometBroadcastHashJoin [rnk,item_sk,i_product_name,i_item_sk,i_product_name] + CometProject [rnk,item_sk,i_product_name] + CometBroadcastHashJoin [item_sk,rnk,item_sk,i_item_sk,i_product_name] + CometProject [item_sk,rnk,item_sk] + CometSortMergeJoin [item_sk,rnk,item_sk,rnk] + CometSort [item_sk,rnk] + CometColumnarExchange [rnk] #1 + WholeStageCodegen (3) + Project [item_sk,rnk] + Filter [rnk,item_sk] + InputAdapter + Window [rank_col] + WindowGroupLimit [rank_col] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometColumnarExchange #2 WindowGroupLimit [rank_col] - WholeStageCodegen (2) - Sort [rank_col] - Filter [rank_col] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count] - InputAdapter - Exchange [ss_store_sk] #4 - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] - ColumnarToRow - InputAdapter + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [ss_store_sk,sum,count] [rank_col,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_store_sk] #4 + CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] CometProject [ss_store_sk,ss_net_profit] CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (1) - HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (10) - Sort [rnk] - InputAdapter - Exchange [rnk] #5 - WholeStageCodegen (9) - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WindowGroupLimit [rank_col] - WholeStageCodegen (8) - Sort [rank_col] - InputAdapter - Exchange #6 + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_item_sk] #3 + CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometSort [item_sk,rnk] + CometColumnarExchange [rnk] #5 + WholeStageCodegen (6) + Project [item_sk,rnk] + Filter [rnk,item_sk] + InputAdapter + Window [rank_col] + WindowGroupLimit [rank_col] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometColumnarExchange #6 WindowGroupLimit [rank_col] - WholeStageCodegen (7) - Sort [rank_col] - Filter [rank_col] - ReusedSubquery [rank_col] #1 - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - InputAdapter - ReusedExchange [ss_item_sk,sum,count] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometFilter [i_item_sk,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_product_name] - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + ReusedSubquery [rank_col] #1 + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + ReusedExchange [ss_item_sk,sum,count] #3 + CometBroadcastExchange [i_item_sk,i_product_name] #7 + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_product_name] + ReusedExchange [i_item_sk,i_product_name] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/explain.txt index acee604c97..4ff7fc2a1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/explain.txt @@ -1,53 +1,63 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`item` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +TakeOrderedAndProject (39) ++- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * Project (34) + +- * Filter (33) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (32) + :- * CometColumnarToRow (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- BroadcastExchange (31) + +- * CometColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) + +- CometNativeScan parquet spark_catalog.default.item (27) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Arguments: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#5)] +PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer` +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [c_customer_sk#6, c_current_addr_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [c_customer_sk#6, c_current_addr_sk#7] @@ -66,128 +76,151 @@ Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Condition : isnotnull(ca_address_sk#8) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#10, 10)) AS ca_zip#11] + +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] -(11) CometBroadcastHashJoin +(12) CometBroadcastHashJoin Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight -(12) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11, d_year#12, d_qoy#13] - -(14) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) - -(15) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#5], [d_date_sk#11], Inner, BuildRight - -(18) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(20) CometFilter -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) - -(21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(22) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Right output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [ws_item_sk#2], [i_item_sk#14], Inner, BuildRight - -(23) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] -Arguments: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15], [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(24) ColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(25) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#17] - -(26) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) - -(27) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_id#17], [i_item_id#17] - -(28) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#17] - -(29) BroadcastExchange -Input [1]: [i_item_id#17] +(13) CometProject +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) + +(16) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] + +(18) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight + +(19) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] + +(20) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) + +(22) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] + +(24) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] + +(27) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] +ReadSchema: struct + +(28) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : i_item_sk#18 IN (2,3,5,7,11,13,17,19,23,29) + +(29) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_id#20], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] + +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [i_item_id#20] + +(31) BroadcastExchange +Input [1]: [i_item_id#20] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] -(30) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] +(32) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#20] Join type: ExistenceJoin(exists#1) Join condition: None -(31) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +(33) Filter [codegen id : 2] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) -(32) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] +(34) Project [codegen id : 2] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] -(33) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] +(35) HashAggregate [codegen id : 2] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] +Aggregate Attributes [1]: [sum#21] +Results [3]: [ca_zip#11, ca_city#9, sum#22] + +(36) CometColumnarExchange +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(34) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(37) CometColumnarToRow [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] -(35) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] +(38) HashAggregate [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#23] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#23,17,2) AS sum(ws_sales_price)#24] -(36) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] +(39) TakeOrderedAndProject +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/extended.txt new file mode 100644 index 0000000000..8fb8fa7188 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/extended.txt @@ -0,0 +1,41 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- Filter + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 29 out of 36 eligible operators (80%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/simplified.txt index 68d48dbfa2..fd8a442c1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/simplified.txt @@ -1,43 +1,46 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + CometColumnarToRow + InputAdapter + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 + CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/explain.txt index c287a20249..14844a8158 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) +TakeOrderedAndProject (43) ++- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) : :- * HashAggregate (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- * ColumnarToRow (26) + : : +- * CometColumnarToRow (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) : : :- CometProject (20) @@ -17,43 +17,51 @@ TakeOrderedAndProject (42) : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : : : +- CometNativeScan parquet spark_catalog.default.store (9) : : : +- CometBroadcastExchange (18) : : : +- CometProject (17) : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (30) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.customer_address (21) + : +- BroadcastExchange (34) + : +- * CometColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometNativeScan parquet spark_catalog.default.customer (30) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometFilter (38) + +- CometNativeScan parquet spark_catalog.default.customer_address (37) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] @@ -76,9 +84,12 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_city#13] -Arguments: [s_store_sk#12, s_city#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#12, s_city#13] @@ -101,9 +112,12 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Arguments: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] @@ -126,9 +140,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(21) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ca_address_sk#17, ca_city#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct (22) CometFilter Input [2]: [ca_address_sk#17, ca_city#18] @@ -147,78 +164,86 @@ Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] -(26) ColumnarToRow [codegen id : 1] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] - -(27) HashAggregate [codegen id : 1] +(26) CometHashAggregate Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#19, sum#20] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -(28) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(27) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(29) HashAggregate [codegen id : 4] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] +(28) CometColumnarToRow [codegen id : 3] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] + +(29) HashAggregate [codegen id : 3] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#23, sum(UnscaledValue(ss_net_profit#7))#24] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#25, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#23,17,2) AS amt#26, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#24,17,2) AS profit#27] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#21, sum(UnscaledValue(ss_net_profit#7))#22] +Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#23, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#21,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#22,17,2) AS profit#25] -(30) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(30) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (31) CometFilter -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_current_addr_sk#29)) +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) + +(32) CometProject +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#26, c_current_addr_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#31] -(32) ColumnarToRow [codegen id : 2] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] -(33) BroadcastExchange -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(34) BroadcastExchange +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(34) BroadcastHashJoin [codegen id : 4] +(35) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#28] +Right keys [1]: [c_customer_sk#26] Join type: Inner Join condition: None -(35) Project [codegen id : 4] -Output [7]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27, c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(36) Project [codegen id : 3] +Output [7]: [ss_ticket_number#5, bought_city#23, amt#24, profit#25, c_current_addr_sk#27, c_first_name#30, c_last_name#31] +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#23, amt#24, profit#25, c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(37) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#32, ca_city#33] -Arguments: [ca_address_sk#32, ca_city#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct -(37) CometFilter +(38) CometFilter Input [2]: [ca_address_sk#32, ca_city#33] Condition : (isnotnull(ca_address_sk#32) AND isnotnull(ca_city#33)) -(38) ColumnarToRow [codegen id : 3] +(39) CometColumnarToRow [codegen id : 2] Input [2]: [ca_address_sk#32, ca_city#33] -(39) BroadcastExchange +(40) BroadcastExchange Input [2]: [ca_address_sk#32, ca_city#33] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(40) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [c_current_addr_sk#29] +(41) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [c_current_addr_sk#27] Right keys [1]: [ca_address_sk#32] Join type: Inner -Join condition: NOT (ca_city#33 = bought_city#25) +Join condition: NOT (ca_city#33 = bought_city#23) -(41) Project [codegen id : 4] -Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Input [9]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] +(42) Project [codegen id : 3] +Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] +Input [9]: [ss_ticket_number#5, bought_city#23, amt#24, profit#25, c_current_addr_sk#27, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] -(42) TakeOrderedAndProject -Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#25 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] +(43) TakeOrderedAndProject +Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] +Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#23 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/extended.txt new file mode 100644 index 0000000000..2e15a40e62 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/extended.txt @@ -0,0 +1,45 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_ticket_number#1, ss_customer_sk#2, ca_city#3 AS bought_city#4, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#5))#6,17,2) AS amt#7, MakeDecimal(sum(UnscaledValue(ss_net_profit#8))#9,17,2) AS profit#10)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 32 out of 40 eligible operators (80%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/simplified.txt index 708b8bb3ec..db4cb883e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/simplified.txt @@ -1,52 +1,51 @@ TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] BroadcastHashJoin [ss_customer_sk,c_customer_sk] HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk] #3 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #5 - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometColumnarToRow + InputAdapter + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow + WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/explain.txt index fe442b233c..3e86ba0e55 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/explain.txt @@ -1,241 +1,273 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * Sort (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- Exchange (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- Window (33) - : +- * Sort (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (42) - +- * Project (41) - +- Window (40) - +- * Sort (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight - -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(18) ColumnarToRow [codegen id : 1] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(19) HashAggregate [codegen id : 1] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] - -(20) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(21) HashAggregate [codegen id : 2] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] - -(22) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(23) Sort [codegen id : 3] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 4] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) + +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight + +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] + +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] + +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] + +(22) HashAggregate [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#17] +Results [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS _w0#19] + +(23) CometColumnarExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] (26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] -(27) Filter [codegen id : 13] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) +(29) Filter [codegen id : 10] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) -(28) Project [codegen id : 13] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(30) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -(29) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] +(31) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] -(30) HashAggregate [codegen id : 6] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#29] +(32) CometColumnarToRow [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] -(31) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(33) HashAggregate [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] +Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] +Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] -(32) Sort [codegen id : 7] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 +(34) CometColumnarExchange +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(33) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#30], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +(35) CometSort +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] -(34) Project [codegen id : 8] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#29, rn#30] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29, rn#30] +(36) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] -(35) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#29, rn#30] +(37) Window +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] + +(39) BroadcastExchange +Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(36) BroadcastHashJoin [codegen id : 13] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#30 + 1)] +(40) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] Join type: Inner Join condition: None -(37) Project [codegen id : 13] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#29, rn#30] +(41) Project [codegen id : 10] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] + +(42) ReusedExchange [Reuses operator id: 34] +Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] -(38) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37] +(43) CometSort +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] -(39) Sort [codegen id : 11] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 +(44) CometColumnarToRow [codegen id : 8] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] -(40) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#38], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] +(45) Window +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] -(41) Project [codegen id : 12] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#37, rn#38] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37, rn#38] +(46) Project [codegen id : 9] +Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] -(42) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#37, rn#38] +(47) BroadcastExchange +Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(43) BroadcastHashJoin [codegen id : 13] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#38 - 1)] +(48) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] Join type: Inner Join condition: None -(44) Project [codegen id : 13] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#29 AS psum#39, sum_sales#37 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#37, rn#38] +(49) Project [codegen id : 10] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] -(45) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +(50) TakeOrderedAndProject +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] +Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/extended.txt new file mode 100644 index 0000000000..f63237bcb4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/extended.txt @@ -0,0 +1,93 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#1, i_brand#2, s_store_name#3, s_company_name#4, d_year#5, d_moy#6, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#8,17,2) AS sum_sales#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#8,17,2) AS _w0#10)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#11, i_brand#12, s_store_name#13, s_company_name#14, d_year#15, d_moy#16, MakeDecimal(sum(UnscaledValue(ss_sales_price#17))#8,17,2) AS sum_sales#18)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#11, i_brand#12, s_store_name#13, s_company_name#14, d_year#15, d_moy#16, MakeDecimal(sum(UnscaledValue(ss_sales_price#17))#8,17,2) AS sum_sales#18)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 66 out of 85 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/simplified.txt index 429e65385e..7f8cac7be4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum] - WholeStageCodegen (13) + WholeStageCodegen (10) Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,60 +8,63 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (4) + WholeStageCodegen (3) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (3) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (8) + WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (7) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (12) + WholeStageCodegen (9) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (11) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (8) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/explain.txt index 97925f1aca..7f80dcf38f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/explain.txt @@ -1,45 +1,53 @@ == Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * ColumnarToRow (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometHashAggregate (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (13) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.store (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.customer_address (14) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometNativeScan parquet spark_catalog.default.date_dim (20) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] +ReadSchema: struct (2) CometFilter Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometNativeScan: `spark_catalog`.`default`.`store` +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (4) CometFilter Input [1]: [s_store_sk#8] @@ -58,95 +66,104 @@ Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(8) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (9) CometFilter Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) +Condition : (isnotnull(cd_demo_sk#9) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = 4 yr Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = D) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = 2 yr Degree ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = College )))) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) AS cd_marital_status#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) AS cd_education_status#13] + +(11) CometBroadcastExchange +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] -(11) CometBroadcastHashJoin +(12) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight -(12) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +(13) CometProject +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(13) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] +(14) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct -(14) CometFilter -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) +(15) CometFilter +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (CO,OH,TX) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (OR,MN,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (VA,CA,MS))) -(15) CometProject -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] +(16) CometProject +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) AS ca_state#17] -(16) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] +(17) CometBroadcastExchange +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] -(17) CometBroadcastHashJoin +(18) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#2], [ca_address_sk#12], Inner, ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight -(18) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] +(19) CometProject +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] +(20) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_year#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct -(20) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(21) CometFilter +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) -(21) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] +(22) CometProject +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] -(22) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] +(23) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight -(24) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] +(25) CometProject +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] Arguments: [ss_quantity#4], [ss_quantity#4] -(25) ColumnarToRow [codegen id : 1] -Input [1]: [ss_quantity#4] - -(26) HashAggregate [codegen id : 1] +(26) CometHashAggregate Input [1]: [ss_quantity#4] Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum#17] -Results [1]: [sum#18] -(27) Exchange -Input [1]: [sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] +(27) CometExchange +Input [1]: [sum#20] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 2] -Input [1]: [sum#18] +(28) CometHashAggregate +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum(ss_quantity#4)#19] -Results [1]: [sum(ss_quantity#4)#19 AS sum(ss_quantity)#20] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/extended.txt new file mode 100644 index 0000000000..bdbe1b496a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/simplified.txt index 73c1d61bf6..5feebbec84 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/simplified.txt @@ -1,32 +1,31 @@ -WholeStageCodegen (2) - HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_quantity] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity] - CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] - CometProject [ss_quantity,ss_sold_date_sk] - CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometHashAggregate [sum] [sum(ss_quantity),sum(ss_quantity)] + CometExchange #1 + CometHashAggregate [ss_quantity] [sum] + CometProject [ss_quantity] + CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] + CometProject [ss_quantity,ss_sold_date_sk] + CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/explain.txt index 9aa373d6a8..1d6b3ef43f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/explain.txt @@ -1,54 +1,91 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- Union (41) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * Sort (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- Exchange (18) - : +- * HashAggregate (17) - : +- * ColumnarToRow (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - :- * Project (33) - : +- * Filter (32) - : +- Window (31) - : +- * Sort (30) - : +- Window (29) - : +- * Sort (28) - : +- ReusedExchange (27) - +- * Project (40) - +- * Filter (39) - +- Window (38) - +- * Sort (37) - +- Window (36) - +- * Sort (35) - +- ReusedExchange (34) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] +PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] @@ -62,9 +99,12 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(5) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct (6) CometFilter Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] @@ -83,9 +123,12 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(10) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_moy#14] @@ -108,130 +151,275 @@ Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -(16) ColumnarToRow [codegen id : 1] -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] - -(17) HashAggregate [codegen id : 1] +(16) CometHashAggregate Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(18) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 2] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] -(20) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(21) Sort [codegen id : 3] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] -(23) Sort [codegen id : 4] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 (24) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] - -(25) Filter [codegen id : 5] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) - -(26) Project [codegen id : 5] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] - -(27) ReusedExchange [Reuses operator id: 20] -Output [3]: [item#37, return_ratio#38, currency_ratio#39] - -(28) Sort [codegen id : 8] -Input [3]: [item#37, return_ratio#38, currency_ratio#39] -Arguments: [return_ratio#38 ASC NULLS FIRST], false, 0 - -(29) Window -Input [3]: [item#37, return_ratio#38, currency_ratio#39] -Arguments: [rank(return_ratio#38) windowspecdefinition(return_ratio#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#40], [return_ratio#38 ASC NULLS FIRST] - -(30) Sort [codegen id : 9] -Input [4]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40] -Arguments: [currency_ratio#39 ASC NULLS FIRST], false, 0 - -(31) Window -Input [4]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40] -Arguments: [rank(currency_ratio#39) windowspecdefinition(currency_ratio#39 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#41], [currency_ratio#39 ASC NULLS FIRST] - -(32) Filter [codegen id : 10] -Input [5]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40, currency_rank#41] -Condition : ((return_rank#40 <= 10) OR (currency_rank#41 <= 10)) - -(33) Project [codegen id : 10] -Output [5]: [catalog AS channel#42, item#37, return_ratio#38, return_rank#40, currency_rank#41] -Input [5]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40, currency_rank#41] - -(34) ReusedExchange [Reuses operator id: 20] -Output [3]: [item#43, return_ratio#44, currency_ratio#45] - -(35) Sort [codegen id : 13] -Input [3]: [item#43, return_ratio#44, currency_ratio#45] -Arguments: [return_ratio#44 ASC NULLS FIRST], false, 0 - -(36) Window -Input [3]: [item#43, return_ratio#44, currency_ratio#45] -Arguments: [rank(return_ratio#44) windowspecdefinition(return_ratio#44 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#46], [return_ratio#44 ASC NULLS FIRST] - -(37) Sort [codegen id : 14] -Input [4]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46] -Arguments: [currency_ratio#45 ASC NULLS FIRST], false, 0 - -(38) Window -Input [4]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46] -Arguments: [rank(currency_ratio#45) windowspecdefinition(currency_ratio#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#47], [currency_ratio#45 ASC NULLS FIRST] - -(39) Filter [codegen id : 15] -Input [5]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46, currency_rank#47] -Condition : ((return_rank#46 <= 10) OR (currency_rank#47 <= 10)) - -(40) Project [codegen id : 15] -Output [5]: [store AS channel#48, item#43, return_ratio#44, return_rank#46, currency_rank#47] -Input [5]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46, currency_rank#47] - -(41) Union - -(42) HashAggregate [codegen id : 16] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) + +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] + +(27) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] +PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) + +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(31) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] +PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) + +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(54) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(43) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(44) HashAggregate [codegen id : 17] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -(45) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/extended.txt new file mode 100644 index 0000000000..262571ed59 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/extended.txt @@ -0,0 +1,86 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- Filter + +- Window + +- Sort + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 63 out of 80 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/simplified.txt index 36e05fccb5..117b74a75b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/simplified.txt @@ -1,71 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (17) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (16) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (5) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (4) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (3) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (1) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (10) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (9) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (8) - Sort [return_ratio] - InputAdapter - ReusedExchange [item,return_ratio,currency_ratio] #2 - WholeStageCodegen (15) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (14) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (13) - Sort [return_ratio] - InputAdapter - ReusedExchange [item,return_ratio,currency_ratio] #2 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/explain.txt index b4ec3d7599..4adaf7e34f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/explain.txt @@ -1,60 +1,87 @@ == Physical Plan == -TakeOrderedAndProject (51) -+- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * Expand (47) - +- Union (46) - :- * HashAggregate (22) - : +- Exchange (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - :- * HashAggregate (24) - : +- ReusedExchange (23) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * ColumnarToRow (42) - +- CometProject (41) - +- CometBroadcastHashJoin (40) - :- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometUnion (35) - : : :- CometProject (27) - : : : +- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (25) - : : +- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometBroadcastExchange (29) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (28) - : : +- CometProject (32) - : : +- CometFilter (31) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (30) - : +- ReusedExchange (36) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (74) ++- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Expand (69) + +- Union (68) + :- * HashAggregate (23) + : +- * CometColumnarToRow (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.store (14) + :- * HashAggregate (43) + : +- * CometColumnarToRow (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (33) + : : +- CometBroadcastHashJoin (32) + : : :- CometUnion (30) + : : : :- CometProject (26) + : : : : +- CometFilter (25) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (24) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (31) + : +- CometBroadcastExchange (37) + : +- CometProject (36) + : +- CometFilter (35) + : +- CometNativeScan parquet spark_catalog.default.catalog_page (34) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometUnion (54) + : : :- CometProject (46) + : : : +- CometFilter (45) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (44) + : : +- CometProject (53) + : : +- CometBroadcastHashJoin (52) + : : :- CometBroadcastExchange (48) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (47) + : : +- CometProject (51) + : : +- CometFilter (50) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (49) + : +- ReusedExchange (55) + +- CometBroadcastExchange (61) + +- CometProject (60) + +- CometFilter (59) + +- CometNativeScan parquet spark_catalog.default.web_site (58) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] @@ -64,9 +91,13 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] @@ -80,9 +111,12 @@ Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, n Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21, d_date#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [d_date_sk#21, d_date#22] @@ -105,173 +139,290 @@ Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(14) CometNativeScan: `spark_catalog`.`default`.`store` +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (15) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(16) CometBroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#25] -(17) CometBroadcastHashJoin +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] +Right output [2]: [s_store_sk#23, s_store_id#25] Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] - -(19) ColumnarToRow [codegen id : 1] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(20) HashAggregate [codegen id : 1] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -(21) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] -(22) HashAggregate [codegen id : 2] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#39, store channel AS channel#40, concat(store, s_store_id#24) AS id#41] - -(23) ReusedExchange [Reuses operator id: 21] -Output [5]: [cp_catalog_page_id#42, sum#43, sum#44, sum#45, sum#46] - -(24) HashAggregate [codegen id : 4] -Input [5]: [cp_catalog_page_id#42, sum#43, sum#44, sum#45, sum#46] -Keys [1]: [cp_catalog_page_id#42] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#48)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#51, sum(UnscaledValue(return_amt#48))#52, sum(UnscaledValue(profit#49))#53, sum(UnscaledValue(net_loss#50))#54] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#47))#51,17,2) AS sales#55, MakeDecimal(sum(UnscaledValue(return_amt#48))#52,17,2) AS returns#56, (MakeDecimal(sum(UnscaledValue(profit#49))#53,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#54,17,2)) AS profit#57, catalog channel AS channel#58, concat(catalog_page, cp_catalog_page_id#42) AS id#59] - -(25) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] -Arguments: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] - -(26) CometFilter -Input [4]: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] -Condition : isnotnull(ws_web_site_sk#60) - -(27) CometProject -Input [4]: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] -Arguments: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69], [ws_web_site_sk#60 AS wsr_web_site_sk#64, ws_sold_date_sk#63 AS date_sk#65, ws_ext_sales_price#61 AS sales_price#66, ws_net_profit#62 AS profit#67, 0.00 AS return_amt#68, 0.00 AS net_loss#69] - -(28) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Arguments: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] - -(29) CometBroadcastExchange -Input [5]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Arguments: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] - -(30) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] - -(31) CometFilter -Input [4]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] -Condition : ((isnotnull(ws_item_sk#75) AND isnotnull(ws_order_number#77)) AND isnotnull(ws_web_site_sk#76)) - -(32) CometProject -Input [4]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77], [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77] - -(33) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Right output [3]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77] -Arguments: [wr_item_sk#70, wr_order_number#71], [ws_item_sk#75, ws_order_number#77], Inner, BuildLeft - -(34) CometProject -Input [8]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74, ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77] -Arguments: [wsr_web_site_sk#79, date_sk#80, sales_price#81, profit#82, return_amt#83, net_loss#84], [ws_web_site_sk#76 AS wsr_web_site_sk#79, wr_returned_date_sk#74 AS date_sk#80, 0.00 AS sales_price#81, 0.00 AS profit#82, wr_return_amt#72 AS return_amt#83, wr_net_loss#73 AS net_loss#84] - -(35) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69] -Child 1 Input [6]: [wsr_web_site_sk#79, date_sk#80, sales_price#81, profit#82, return_amt#83, net_loss#84] - -(36) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#85] - -(37) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69] -Right output [1]: [d_date_sk#85] -Arguments: [date_sk#65], [d_date_sk#85], Inner, BuildRight - -(38) CometProject -Input [7]: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69, d_date_sk#85] -Arguments: [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69], [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69] - -(39) ReusedExchange [Reuses operator id: 16] -Output [2]: [web_site_sk#86, web_site_id#87] - -(40) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69] -Right output [2]: [web_site_sk#86, web_site_id#87] -Arguments: [wsr_web_site_sk#64], [web_site_sk#86], Inner, BuildRight - -(41) CometProject -Input [7]: [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_sk#86, web_site_id#87] -Arguments: [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87], [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87] - -(42) ColumnarToRow [codegen id : 5] -Input [5]: [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87] - -(43) HashAggregate [codegen id : 5] -Input [5]: [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87] -Keys [1]: [web_site_id#87] -Functions [4]: [partial_sum(UnscaledValue(sales_price#66)), partial_sum(UnscaledValue(return_amt#68)), partial_sum(UnscaledValue(profit#67)), partial_sum(UnscaledValue(net_loss#69))] -Aggregate Attributes [4]: [sum#88, sum#89, sum#90, sum#91] -Results [5]: [web_site_id#87, sum#92, sum#93, sum#94, sum#95] - -(44) Exchange -Input [5]: [web_site_id#87, sum#92, sum#93, sum#94, sum#95] -Arguments: hashpartitioning(web_site_id#87, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(45) HashAggregate [codegen id : 6] -Input [5]: [web_site_id#87, sum#92, sum#93, sum#94, sum#95] -Keys [1]: [web_site_id#87] -Functions [4]: [sum(UnscaledValue(sales_price#66)), sum(UnscaledValue(return_amt#68)), sum(UnscaledValue(profit#67)), sum(UnscaledValue(net_loss#69))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#66))#96, sum(UnscaledValue(return_amt#68))#97, sum(UnscaledValue(profit#67))#98, sum(UnscaledValue(net_loss#69))#99] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#66))#96,17,2) AS sales#100, MakeDecimal(sum(UnscaledValue(return_amt#68))#97,17,2) AS returns#101, (MakeDecimal(sum(UnscaledValue(profit#67))#98,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#69))#99,17,2)) AS profit#102, web channel AS channel#103, concat(web_site, web_site_id#87) AS id#104] - -(46) Union - -(47) Expand [codegen id : 7] -Input [5]: [sales#37, returns#38, profit#39, channel#40, id#41] -Arguments: [[sales#37, returns#38, profit#39, channel#40, id#41, 0], [sales#37, returns#38, profit#39, channel#40, null, 1], [sales#37, returns#38, profit#39, null, null, 3]], [sales#37, returns#38, profit#39, channel#105, id#106, spark_grouping_id#107] - -(48) HashAggregate [codegen id : 7] -Input [6]: [sales#37, returns#38, profit#39, channel#105, id#106, spark_grouping_id#107] -Keys [3]: [channel#105, id#106, spark_grouping_id#107] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] -Results [9]: [channel#105, id#106, spark_grouping_id#107, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] - -(49) Exchange -Input [9]: [channel#105, id#106, spark_grouping_id#107, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] -Arguments: hashpartitioning(channel#105, id#106, spark_grouping_id#107, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(50) HashAggregate [codegen id : 8] -Input [9]: [channel#105, id#106, spark_grouping_id#107, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] -Keys [3]: [channel#105, id#106, spark_grouping_id#107] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#120, sum(returns#38)#121, sum(profit#39)#122] -Results [5]: [channel#105, id#106, sum(sales#37)#120 AS sales#123, sum(returns#38)#121 AS returns#124, sum(profit#39)#122 AS profit#125] - -(51) TakeOrderedAndProject -Input [5]: [channel#105, id#106, sales#123, returns#124, profit#125] -Arguments: 100, [channel#105 ASC NULLS FIRST, id#106 ASC NULLS FIRST], [channel#105, id#106, sales#123, returns#124, profit#125] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#35, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#36, store channel AS channel#37, concat(store, s_store_id#25) AS id#38] + +(24) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +PushedFilters: [IsNotNull(cs_catalog_page_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) + +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] + +(27) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] +PushedFilters: [IsNotNull(cr_catalog_page_sk)] +ReadSchema: struct + +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) + +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] + +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] + +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] + +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight + +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] + +(34) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#61, 16)) AS cp_catalog_page_id#62] + +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] + +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight + +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] + +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] + +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] + +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#62) AS id#75] + +(44) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] +PushedFilters: [IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) + +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] + +(47) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] +ReadSchema: struct + +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] + +(49) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) + +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] + +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft + +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] + +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] + +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] + +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight + +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] + +(58) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) + +(60) CometProject +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#103, 16)) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#113, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#114, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#115, web channel AS channel#116, concat(web_site, web_site_id#104) AS id#117] + +(68) Union + +(69) Expand [codegen id : 4] +Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] +Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] + +(70) HashAggregate [codegen id : 4] +Input [6]: [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] +Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(71) CometColumnarExchange +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(72) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(73) HashAggregate [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] +Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] +Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] + +(74) TakeOrderedAndProject +Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] +Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/extended.txt new file mode 100644 index 0000000000..d2214cd42b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/extended.txt @@ -0,0 +1,82 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(MakeDecimal(sum(UnscaledValue(sales_price#1))#2,17,2) AS sales#3, MakeDecimal(sum(UnscaledValue(return_amt#4))#5,17,2) AS returns#6, (MakeDecimal(sum(UnscaledValue(profit#7))#8,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#9))#10,17,2)) AS profit#11, store channel AS channel#12, concat(store, s_store_id#13) AS id#14)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(MakeDecimal(sum(UnscaledValue(sales_price#15))#16,17,2) AS sales#17, MakeDecimal(sum(UnscaledValue(return_amt#18))#19,17,2) AS returns#20, (MakeDecimal(sum(UnscaledValue(profit#21))#22,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#23))#24,17,2)) AS profit#25, catalog channel AS channel#26, concat(catalog_page, cp_catalog_page_id#27) AS id#28)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_page + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(MakeDecimal(sum(UnscaledValue(sales_price#29))#30,17,2) AS sales#31, MakeDecimal(sum(UnscaledValue(return_amt#32))#33,17,2) AS returns#34, (MakeDecimal(sum(UnscaledValue(profit#35))#36,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#37))#38,17,2)) AS profit#39, web channel AS channel#40, concat(web_site, web_site_id#41) AS id#42)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 68 out of 76 eligible operators (89%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/simplified.txt index 11227fb08a..5992a2d488 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/simplified.txt @@ -1,65 +1,84 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (8) + WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (1) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - ReusedExchange [cp_catalog_page_id,sum,sum,sum,sum] #2 - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #5 - WholeStageCodegen (5) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #6 - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [web_site_sk,web_site_id] #4 + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #5 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #7 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/explain.txt index 5260461358..09aaf0819c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/explain.txt @@ -1,45 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (28) -+- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.store (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#9)] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -58,95 +67,104 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_numb Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -(8) CometNativeScan: `spark_catalog`.`default`.`store` +(8) CometNativeScan parquet spark_catalog.default.store Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (9) CometFilter Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Condition : isnotnull(s_store_sk#10) -(10) CometBroadcastExchange +(10) CometProject Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_street_type#15, 15)) AS s_street_type#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_suite_number#16, 10)) AS s_suite_number#22, s_city#17, s_county#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#19, 2)) AS s_state#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#20, 10)) AS s_zip#24] + +(11) CometBroadcastExchange +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(11) CometBroadcastHashJoin +(12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(12) CometProject -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(14) CometFilter -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(16) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#21] -Arguments: [ss_sold_date_sk#5], [d_date_sk#21], Inner, BuildRight - -(17) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22, d_year#23, d_moy#24] - -(19) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) - -(20) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(21) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] - -(22) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#9], [d_date_sk#22], Inner, BuildRight - -(23) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(24) ColumnarToRow [codegen id : 1] -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(25) HashAggregate [codegen id : 1] -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(13) CometProject +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [1]: [d_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) + +(16) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(17) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight + +(18) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) + +(21) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] + +(23) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight + +(24) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(25) CometHashAggregate +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#25, sum#26, sum#27, sum#28, sum#29] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -(26) Exchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(26) CometExchange +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 2] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(27) CometHashAggregate +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35 AS 30 days #40, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36 AS 31 - 60 days #41, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37 AS 61 - 90 days #42, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38 AS 91 - 120 days #43, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39 AS >120 days #44] -(28) TakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] -Arguments: 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] +(28) CometTakeOrderedAndProject +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] + +(29) CometColumnarToRow [codegen id : 1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/extended.txt new file mode 100644 index 0000000000..a188fea58c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/simplified.txt index 8b0db61368..e53cdee055 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/simplified.txt @@ -1,32 +1,31 @@ -TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (2) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - WholeStageCodegen (1) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 + CometHashAggregate [sr_returned_date_sk,ss_sold_date_sk] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastExchange [d_date_sk] #4 - CometFilter [d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 + CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastExchange [d_date_sk] #4 + CometFilter [d_date_sk] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/explain.txt index 8dd829b3cc..72bbbaadd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/explain.txt @@ -1,44 +1,67 @@ == Physical Plan == -TakeOrderedAndProject (27) -+- * Filter (26) - +- Window (25) - +- * Sort (24) - +- Exchange (23) - +- * Project (22) - +- * SortMergeJoin FullOuter (21) - :- * Sort (18) - : +- Exchange (17) - : +- * Project (16) - : +- Window (15) - : +- * Sort (14) - : +- Exchange (13) - : +- * HashAggregate (12) - : +- Exchange (11) - : +- * HashAggregate (10) - : +- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- * Sort (20) - +- ReusedExchange (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +TakeOrderedAndProject (43) ++- * Filter (42) + +- Window (41) + +- * CometColumnarToRow (40) + +- CometSort (39) + +- CometExchange (38) + +- CometProject (37) + +- CometSortMergeJoin (36) + :- CometSort (19) + : +- CometColumnarExchange (18) + : +- * Project (17) + : +- Window (16) + : +- * CometColumnarToRow (15) + : +- CometSort (14) + : +- CometColumnarExchange (13) + : +- * HashAggregate (12) + : +- * CometColumnarToRow (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- CometSort (35) + +- CometColumnarExchange (34) + +- * Project (33) + +- Window (32) + +- * CometColumnarToRow (31) + +- CometSort (30) + +- CometColumnarExchange (29) + +- * HashAggregate (28) + +- * CometColumnarToRow (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.store_sales (20) + +- ReusedExchange (22) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] @@ -61,85 +84,151 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] -(9) ColumnarToRow [codegen id : 1] -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] - -(10) HashAggregate [codegen id : 1] +(9) CometHashAggregate Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] -(11) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(11) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] -(12) HashAggregate [codegen id : 2] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] +(12) HashAggregate [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS _w0#11, ws_item_sk#1] - -(13) Exchange -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(14) Sort [codegen id : 3] -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(15) Window -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [sum(_w0#11) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] - -(16) Project [codegen id : 4] -Output [3]: [item_sk#10, d_date#5, cume_sales#12] -Input [5]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1, cume_sales#12] - -(17) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 5] -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(19) ReusedExchange [Reuses operator id: 17] -Output [3]: [item_sk#13, d_date#14, cume_sales#15] - -(20) Sort [codegen id : 10] -Input [3]: [item_sk#13, d_date#14, cume_sales#15] -Arguments: [item_sk#13 ASC NULLS FIRST, d_date#14 ASC NULLS FIRST], false, 0 - -(21) SortMergeJoin [codegen id : 11] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#13, d_date#14] -Join type: FullOuter -Join condition: None - -(22) Project [codegen id : 11] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#13 END AS item_sk#16, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#14 END AS d_date#17, cume_sales#12 AS web_sales#18, cume_sales#15 AS store_sales#19] -Input [6]: [item_sk#10, d_date#5, cume_sales#12, item_sk#13, d_date#14, cume_sales#15] - -(23) Exchange -Input [4]: [item_sk#16, d_date#17, web_sales#18, store_sales#19] -Arguments: hashpartitioning(item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 12] -Input [4]: [item_sk#16, d_date#17, web_sales#18, store_sales#19] -Arguments: [item_sk#16 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(25) Window -Input [4]: [item_sk#16, d_date#17, web_sales#18, store_sales#19] -Arguments: [max(web_sales#18) windowspecdefinition(item_sk#16, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#20, max(store_sales#19) windowspecdefinition(item_sk#16, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#21], [item_sk#16], [d_date#17 ASC NULLS FIRST] - -(26) Filter [codegen id : 13] -Input [6]: [item_sk#16, d_date#17, web_sales#18, store_sales#19, web_cumulative#20, store_cumulative#21] -Condition : ((isnotnull(web_cumulative#20) AND isnotnull(store_cumulative#21)) AND (web_cumulative#20 > store_cumulative#21)) - -(27) TakeOrderedAndProject -Input [6]: [item_sk#16, d_date#17, web_sales#18, store_sales#19, web_cumulative#20, store_cumulative#21] -Arguments: 100, [item_sk#16 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], [item_sk#16, d_date#17, web_sales#18, store_sales#19, web_cumulative#20, store_cumulative#21] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#8] +Results [4]: [ws_item_sk#1 AS item_sk#9, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#8,17,2) AS _w0#10, ws_item_sk#1] + +(13) CometColumnarExchange +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(14) CometSort +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(15) CometColumnarToRow [codegen id : 2] +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] + +(16) Window +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: [sum(_w0#10) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#11], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] + +(17) Project [codegen id : 3] +Output [3]: [item_sk#9, d_date#5, cume_sales#11] +Input [5]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1, cume_sales#11] + +(18) CometColumnarExchange +Input [3]: [item_sk#9, d_date#5, cume_sales#11] +Arguments: hashpartitioning(item_sk#9, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(19) CometSort +Input [3]: [item_sk#9, d_date#5, cume_sales#11] +Arguments: [item_sk#9, d_date#5, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(20) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#14)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Condition : isnotnull(ss_item_sk#12) + +(22) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#15, d_date#16] + +(23) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Right output [2]: [d_date_sk#15, d_date#16] +Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight + +(24) CometProject +Input [5]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#15, d_date#16] +Arguments: [ss_item_sk#12, ss_sales_price#13, d_date#16], [ss_item_sk#12, ss_sales_price#13, d_date#16] + +(25) CometHashAggregate +Input [3]: [ss_item_sk#12, ss_sales_price#13, d_date#16] +Keys [2]: [ss_item_sk#12, d_date#16] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))] + +(26) CometExchange +Input [3]: [ss_item_sk#12, d_date#16, sum#17] +Arguments: hashpartitioning(ss_item_sk#12, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(27) CometColumnarToRow [codegen id : 4] +Input [3]: [ss_item_sk#12, d_date#16, sum#17] + +(28) HashAggregate [codegen id : 4] +Input [3]: [ss_item_sk#12, d_date#16, sum#17] +Keys [2]: [ss_item_sk#12, d_date#16] +Functions [1]: [sum(UnscaledValue(ss_sales_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#13))#18] +Results [4]: [ss_item_sk#12 AS item_sk#19, d_date#16, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#18,17,2) AS _w0#20, ss_item_sk#12] + +(29) CometColumnarExchange +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(30) CometSort +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12], [ss_item_sk#12 ASC NULLS FIRST, d_date#16 ASC NULLS FIRST] + +(31) CometColumnarToRow [codegen id : 5] +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] + +(32) Window +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: [sum(_w0#20) windowspecdefinition(ss_item_sk#12, d_date#16 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#21], [ss_item_sk#12], [d_date#16 ASC NULLS FIRST] + +(33) Project [codegen id : 6] +Output [3]: [item_sk#19, d_date#16, cume_sales#21] +Input [5]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12, cume_sales#21] + +(34) CometColumnarExchange +Input [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: hashpartitioning(item_sk#19, d_date#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(35) CometSort +Input [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#19, d_date#16, cume_sales#21], [item_sk#19 ASC NULLS FIRST, d_date#16 ASC NULLS FIRST] + +(36) CometSortMergeJoin +Left output [3]: [item_sk#9, d_date#5, cume_sales#11] +Right output [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#9, d_date#5], [item_sk#19, d_date#16], FullOuter + +(37) CometProject +Input [6]: [item_sk#9, d_date#5, cume_sales#11, item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#16 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] + +(38) CometExchange +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(39) CometSort +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST] + +(40) CometColumnarToRow [codegen id : 7] +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] + +(41) Window +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] + +(42) Filter [codegen id : 8] +Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Condition : ((isnotnull(web_cumulative#26) AND isnotnull(store_cumulative#27)) AND (web_cumulative#26 > store_cumulative#27)) + +(43) TakeOrderedAndProject +Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Arguments: 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/extended.txt new file mode 100644 index 0000000000..701b464a3f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/extended.txt @@ -0,0 +1,48 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ws_item_sk#1 AS item_sk#2, d_date#3, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#5,17,2) AS _w0#6, ws_item_sk#1)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometColumnarExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_item_sk#7 AS item_sk#8, d_date#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#11,17,2) AS _w0#12, ss_item_sk#7)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 41 eligible operators (78%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/simplified.txt index c6a24b17f2..7641c3af73 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/simplified.txt @@ -1,46 +1,59 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (13) + WholeStageCodegen (8) Filter [web_cumulative,store_cumulative] InputAdapter Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (12) - Sort [item_sk,d_date] + WholeStageCodegen (7) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (11) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (5) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (4) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (3) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #4 - WholeStageCodegen (1) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (10) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,cume_sales] #2 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (3) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometColumnarExchange [ws_item_sk] #3 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #6 + WholeStageCodegen (6) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometColumnarExchange [ss_item_sk] #7 + WholeStageCodegen (4) + HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_item_sk,d_date] #8 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/explain.txt index 48652a0632..6d0b32fc05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/explain.txt @@ -1,28 +1,31 @@ == Physical Plan == TakeOrderedAndProject (19) +- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] @@ -63,41 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnul (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(16) HashAggregate [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(17) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] -(18) HashAggregate [codegen id : 2] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(18) HashAggregate [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] +Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#11 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] (19) TakeOrderedAndProject Input [4]: [d_year#2, brand_id#14, brand#15, ext_price#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/extended.txt new file mode 100644 index 0000000000..a95811b5a9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/extended.txt @@ -0,0 +1,21 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#1, i_brand_id#2 AS brand_id#3, i_brand#4 AS brand#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#7,17,2) AS ext_price#8)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 16 out of 18 eligible operators (88%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/simplified.txt index 2e1cfd5a58..f0cecd2106 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/simplified.txt @@ -1,23 +1,21 @@ TakeOrderedAndProject [d_year,ext_price,brand_id,brand] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (1) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/explain.txt index eabc3792e1..fa3cc01d01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/explain.txt @@ -1,50 +1,58 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- * Filter (27) - +- Window (26) - +- * Sort (25) - +- Exchange (24) - +- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (30) ++- * Project (29) + +- * Filter (28) + +- Window (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Books ,Children ,Electronics ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (personal ,portable ,reference ,self-help )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Women ,Music ,Men ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (accessories ,classical ,fragrances ,pants )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) (3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -63,9 +71,12 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] @@ -88,9 +99,12 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [1]: [s_store_sk#17] @@ -109,48 +123,49 @@ Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] -(20) ColumnarToRow [codegen id : 1] -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] - -(21) HashAggregate [codegen id : 1] +(20) CometHashAggregate Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#19] -(22) Exchange -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(21) CometExchange +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] -(23) HashAggregate [codegen id : 2] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] +(23) HashAggregate [codegen id : 1] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#19] +Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS _w0#21] + +(24) CometColumnarExchange +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(24) Exchange -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(25) CometSort +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [i_manufact_id#5, sum_sales#20, _w0#21], [i_manufact_id#5 ASC NULLS FIRST] -(25) Sort [codegen id : 3] -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 +(26) CometColumnarToRow [codegen id : 2] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] -(26) Window -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#23], [i_manufact_id#5] +(27) Window +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] -(27) Filter [codegen id : 4] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] -Condition : CASE WHEN (avg_quarterly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_quarterly_sales#23)) / avg_quarterly_sales#23) > 0.1000000000000000) ELSE false END +(28) Filter [codegen id : 3] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Condition : CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END -(28) Project [codegen id : 4] -Output [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] +(29) Project [codegen id : 3] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] -(29) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Arguments: 100, [avg_quarterly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] +(30) TakeOrderedAndProject +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/extended.txt new file mode 100644 index 0000000000..3e06441a69 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/extended.txt @@ -0,0 +1,32 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_manufact_id#1, MakeDecimal(sum(UnscaledValue(ss_sales_price#2))#3,17,2) AS sum_sales#4, MakeDecimal(sum(UnscaledValue(ss_sales_price#2))#3,17,2) AS _w0#5)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 23 out of 28 eligible operators (82%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/simplified.txt index ca0948944d..fe2fa1a6ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/simplified.txt @@ -1,37 +1,36 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [i_manufact_id,sum_sales,avg_quarterly_sales] Filter [avg_quarterly_sales,sum_sales] InputAdapter Window [_w0,i_manufact_id] - WholeStageCodegen (3) - Sort [i_manufact_id] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (2) - HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manufact_id,d_qoy] #2 - WholeStageCodegen (1) - HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_manufact_id,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manufact_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_qoy] #4 - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] + CometSort [i_manufact_id,sum_sales,_w0] + CometColumnarExchange [i_manufact_id] #1 + WholeStageCodegen (1) + HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id,d_qoy] #2 + CometHashAggregate [ss_sales_price] [i_manufact_id,d_qoy,sum] + CometProject [i_manufact_id,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manufact_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_qoy] #4 + CometProject [d_date_sk,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/explain.txt index 9204cb2bde..a1594be51f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/explain.txt @@ -1,28 +1,31 @@ == Physical Plan == TakeOrderedAndProject (19) +- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) + +- CometNativeScan parquet spark_catalog.default.item (9) -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] @@ -32,9 +35,13 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] @@ -53,9 +60,12 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] @@ -63,41 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnu (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) ColumnarToRow [codegen id : 1] -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(16) HashAggregate [codegen id : 1] -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [3]: [i_brand#9, i_brand_id#8, sum#12] -(17) Exchange -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] -(18) HashAggregate [codegen id : 2] -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Keys [2]: [i_brand#9, i_brand_id#8] +(18) HashAggregate [codegen id : 1] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] +Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#11 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] (19) TakeOrderedAndProject Input [3]: [brand_id#14, brand#15, ext_price#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/extended.txt new file mode 100644 index 0000000000..13fb0d6edf --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/extended.txt @@ -0,0 +1,21 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_brand_id#1 AS brand_id#2, i_brand#3 AS brand#4, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#6,17,2) AS ext_price#7)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 16 out of 18 eligible operators (88%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/simplified.txt index 93d3c9b4fb..1ae0ac5d4c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/simplified.txt @@ -1,23 +1,21 @@ TakeOrderedAndProject [ext_price,brand_id,brand] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id] #1 - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometColumnarToRow + InputAdapter + CometExchange [i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/explain.txt index c9298fb775..9ddb2d0c60 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/explain.txt @@ -1,67 +1,89 @@ == Physical Plan == -TakeOrderedAndProject (50) -+- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- Union (46) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometProject (33) - : : : +- CometBroadcastHashJoin (32) - : : : :- CometFilter (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : : : +- ReusedExchange (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- * HashAggregate (45) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (65) ++- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometColumnarExchange (62) + +- * HashAggregate (61) + +- Union (60) + :- * HashAggregate (29) + : +- * CometColumnarToRow (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- * HashAggregate (44) + : +- * CometColumnarToRow (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (34) + : : : +- CometBroadcastHashJoin (33) + : : : :- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (30) + : : : +- ReusedExchange (32) + : : +- ReusedExchange (35) + : +- ReusedExchange (38) + +- * HashAggregate (59) + +- * CometColumnarToRow (58) + +- CometExchange (57) + +- CometHashAggregate (56) + +- CometProject (55) + +- CometBroadcastHashJoin (54) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (49) + : : +- CometBroadcastHashJoin (48) + : : :- CometFilter (46) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (45) + : : +- ReusedExchange (47) + : +- ReusedExchange (50) + +- ReusedExchange (53) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] @@ -84,9 +106,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] @@ -109,165 +134,234 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometNativeScan: `spark_catalog`.`default`.`item` +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12, i_color#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct (18) CometFilter Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) +Condition : static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#13, 20)) IN (slate ,blanched ,burnished ) (19) CometProject Input [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12], [i_item_id#12] +Arguments: [i_item_id#14], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight +Right output [1]: [i_item_id#14] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16))], [i_item_id#14], LeftSemi, BuildRight -(22) CometBroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16)) AS i_item_id#15] -(23) CometBroadcastHashJoin +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] +Right output [2]: [i_item_sk#10, i_item_id#15] Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] - -(25) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(26) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] -(27) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] -(28) HashAggregate [codegen id : 2] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] +(29) HashAggregate [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#22] - -(32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] - -(34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#23] - -(35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#23] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#23], Inner, BuildRight - -(36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] - -(37) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#24, i_item_id#25] - -(38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#24, i_item_id#25] -Arguments: [cs_item_sk#19], [i_item_sk#24], Inner, BuildRight - -(39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] -Arguments: [cs_ext_sales_price#20, i_item_id#25], [cs_ext_sales_price#20, i_item_id#25] - -(40) ColumnarToRow [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] - -(41) HashAggregate [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(42) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(43) HashAggregate [codegen id : 4] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(44) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_id#30, sum#31] - -(45) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#30, sum#31] -Keys [1]: [i_item_id#30] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#33] -Results [2]: [i_item_id#30, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#33,17,2) AS total_sales#34] - -(46) Union - -(47) HashAggregate [codegen id : 7] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [i_item_id#11, sum#37, isEmpty#38] - -(48) Exchange -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(49) HashAggregate [codegen id : 8] -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#39] -Results [2]: [i_item_id#11, sum(total_sales#17)#39 AS total_sales#40] - -(50) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#40] -Arguments: 100, [total_sales#40 ASC NULLS FIRST], [i_item_id#11, total_sales#40] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] +Results [2]: [i_item_id#15 AS i_item_id#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#19] + +(30) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#23)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Condition : (isnotnull(cs_bill_addr_sk#20) AND isnotnull(cs_item_sk#21)) + +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#24] + +(33) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Right output [1]: [d_date_sk#24] +Arguments: [cs_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight + +(34) CometProject +Input [5]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, d_date_sk#24] +Arguments: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22], [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] + +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#25] + +(36) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] +Right output [1]: [ca_address_sk#25] +Arguments: [cs_bill_addr_sk#20], [ca_address_sk#25], Inner, BuildRight + +(37) CometProject +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, ca_address_sk#25] +Arguments: [cs_item_sk#21, cs_ext_sales_price#22], [cs_item_sk#21, cs_ext_sales_price#22] + +(38) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#26, i_item_id#27] + +(39) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#21, cs_ext_sales_price#22] +Right output [2]: [i_item_sk#26, i_item_id#27] +Arguments: [cs_item_sk#21], [i_item_sk#26], Inner, BuildRight + +(40) CometProject +Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, i_item_sk#26, i_item_id#27] +Arguments: [cs_ext_sales_price#22, i_item_id#27], [cs_ext_sales_price#22, i_item_id#27] + +(41) CometHashAggregate +Input [2]: [cs_ext_sales_price#22, i_item_id#27] +Keys [1]: [i_item_id#27] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))] + +(42) CometExchange +Input [2]: [i_item_id#27, sum#28] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(43) CometColumnarToRow [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] + +(44) HashAggregate [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] +Keys [1]: [i_item_id#27] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#29] +Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#29,17,2) AS total_sales#30] + +(45) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(46) CometFilter +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : (isnotnull(ws_bill_addr_sk#32) AND isnotnull(ws_item_sk#31)) + +(47) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#35] + +(48) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [1]: [d_date_sk#35] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight + +(49) CometProject +Input [5]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35] +Arguments: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33], [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] + +(50) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#36] + +(51) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] +Right output [1]: [ca_address_sk#36] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#36], Inner, BuildRight + +(52) CometProject +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ca_address_sk#36] +Arguments: [ws_item_sk#31, ws_ext_sales_price#33], [ws_item_sk#31, ws_ext_sales_price#33] + +(53) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#37, i_item_id#38] + +(54) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#31, ws_ext_sales_price#33] +Right output [2]: [i_item_sk#37, i_item_id#38] +Arguments: [ws_item_sk#31], [i_item_sk#37], Inner, BuildRight + +(55) CometProject +Input [4]: [ws_item_sk#31, ws_ext_sales_price#33, i_item_sk#37, i_item_id#38] +Arguments: [ws_ext_sales_price#33, i_item_id#38], [ws_ext_sales_price#33, i_item_id#38] + +(56) CometHashAggregate +Input [2]: [ws_ext_sales_price#33, i_item_id#38] +Keys [1]: [i_item_id#38] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] + +(57) CometExchange +Input [2]: [i_item_id#38, sum#39] +Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(58) CometColumnarToRow [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] + +(59) HashAggregate [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] +Keys [1]: [i_item_id#38] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#40] +Results [2]: [i_item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#40,17,2) AS total_sales#41] + +(60) Union + +(61) HashAggregate [codegen id : 4] +Input [2]: [i_item_id#18, total_sales#19] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(total_sales#19)] +Aggregate Attributes [2]: [sum#42, isEmpty#43] +Results [3]: [i_item_id#18, sum#44, isEmpty#45] + +(62) CometColumnarExchange +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(63) CometColumnarToRow [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] + +(64) HashAggregate [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(total_sales#19)] +Aggregate Attributes [1]: [sum(total_sales#19)#46] +Results [2]: [i_item_id#18, sum(total_sales#19)#46 AS total_sales#47] + +(65) TakeOrderedAndProject +Input [2]: [i_item_id#18, total_sales#47] +Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_item_id#18, total_sales#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/extended.txt new file mode 100644 index 0000000000..a65cb49e1b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/extended.txt @@ -0,0 +1,95 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#1 AS i_item_id#2, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#4,17,2) AS total_sales#5)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#6, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#7))#8,17,2) AS total_sales#9)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#10, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#11))#12,17,2) AS total_sales#13)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 82 out of 89 eligible operators (92%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/simplified.txt index cdddff0dcb..177e910d66 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/simplified.txt @@ -1,64 +1,75 @@ TakeOrderedAndProject [total_sales,i_item_id] - WholeStageCodegen (8) + WholeStageCodegen (5) HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_item_id] #1 + WholeStageCodegen (4) + HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),i_item_id,total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] + CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_color] - WholeStageCodegen (4) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (3) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - ReusedExchange [i_item_id,sum] #2 + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_color] + WholeStageCodegen (2) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + WholeStageCodegen (3) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/explain.txt index 564f4218e9..1549c7702b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/explain.txt @@ -1,241 +1,273 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * Sort (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- Exchange (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (13) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- Window (33) - : +- * Sort (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (42) - +- * Project (41) - +- Window (40) - +- * Sort (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(18) ColumnarToRow [codegen id : 1] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(19) HashAggregate [codegen id : 1] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] - -(20) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(21) HashAggregate [codegen id : 2] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] - -(22) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(23) Sort [codegen id : 3] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 4] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight + +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] + +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] + +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] + +(22) HashAggregate [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#16] +Results [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS _w0#18] + +(23) CometColumnarExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] (26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#14, d_year#11] -(27) Filter [codegen id : 13] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] -Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) +(29) Filter [codegen id : 10] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(28) Project [codegen id : 13] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(30) Project [codegen id : 10] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -(29) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] +(31) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] -(30) HashAggregate [codegen id : 6] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#27] +(32) CometColumnarToRow [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] -(31) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(33) HashAggregate [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] +Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] +Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] +Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] -(32) Sort [codegen id : 7] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 +(34) CometColumnarExchange +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(33) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) CometSort +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] -(34) Project [codegen id : 8] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#27, rn#28] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27, rn#28] +(36) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] -(35) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#27, rn#28] +(37) Window +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] + +(39) BroadcastExchange +Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(36) BroadcastHashJoin [codegen id : 13] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#28 + 1)] +(40) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] Join type: Inner Join condition: None -(37) Project [codegen id : 13] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#27] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#27, rn#28] +(41) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] + +(42) ReusedExchange [Reuses operator id: 34] +Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] -(38) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34] +(43) CometSort +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(39) Sort [codegen id : 11] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 +(44) CometColumnarToRow [codegen id : 8] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] -(40) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] +(45) Window +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(41) Project [codegen id : 12] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#34, rn#35] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34, rn#35] +(46) Project [codegen id : 9] +Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] -(42) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#34, rn#35] +(47) BroadcastExchange +Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(43) BroadcastHashJoin [codegen id : 13] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#35 - 1)] +(48) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] Join type: Inner Join condition: None -(44) Project [codegen id : 13] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#27 AS psum#36, sum_sales#34 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#27, i_category#29, i_brand#30, cc_name#31, sum_sales#34, rn#35] +(49) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] -(45) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +(50) TakeOrderedAndProject +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/extended.txt new file mode 100644 index 0000000000..7e041c6501 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/extended.txt @@ -0,0 +1,93 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#1, i_brand#2, cc_name#3, d_year#4, d_moy#5, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#7,17,2) AS sum_sales#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#7,17,2) AS _w0#9)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.call_center + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#10, i_brand#11, cc_name#12, d_year#13, d_moy#14, MakeDecimal(sum(UnscaledValue(cs_sales_price#15))#7,17,2) AS sum_sales#16)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#10, i_brand#11, cc_name#12, d_year#13, d_moy#14, MakeDecimal(sum(UnscaledValue(cs_sales_price#15))#7,17,2) AS sum_sales#16)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 66 out of 85 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/simplified.txt index c3152d8cf9..3254f03529 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum] - WholeStageCodegen (13) + WholeStageCodegen (10) Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,60 +8,63 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (4) + WholeStageCodegen (3) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (3) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,cc_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 - CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #5 - CometFilter [cc_call_center_sk,cc_name] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (8) + WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (7) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,cc_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (12) + WholeStageCodegen (9) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (11) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (8) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/explain.txt index cfeddd402c..0bbc59c6de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/explain.txt @@ -1,232 +1,268 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * HashAggregate (11) - : : : +- Exchange (10) - : : : +- * HashAggregate (9) - : : : +- * ColumnarToRow (8) - : : : +- CometProject (7) - : : : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (47) ++- * Project (46) + +- * BroadcastHashJoin Inner BuildRight (45) + :- * Project (26) + : +- * BroadcastHashJoin Inner BuildRight (25) + : :- * Project (19) + : : +- * BroadcastHashJoin Inner BuildRight (18) + : : :- * HashAggregate (12) + : : : +- * CometColumnarToRow (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- BroadcastExchange (15) - : : +- * ColumnarToRow (14) - : : +- CometFilter (13) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (12) - : +- BroadcastExchange (22) - : +- * ColumnarToRow (21) - : +- CometProject (20) - : +- CometFilter (19) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) - +- BroadcastExchange (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (32) - : +- * BroadcastHashJoin Inner BuildRight (31) - : :- * HashAggregate (26) - : : +- ReusedExchange (25) - : +- BroadcastExchange (30) - : +- * ColumnarToRow (29) - : +- CometFilter (28) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (27) - +- BroadcastExchange (37) - +- * ColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (17) + : : +- * CometColumnarToRow (16) + : : +- CometProject (15) + : : +- CometFilter (14) + : : +- CometNativeScan parquet spark_catalog.default.store (13) + : +- BroadcastExchange (24) + : +- * CometColumnarToRow (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.date_dim (20) + +- BroadcastExchange (44) + +- * Project (43) + +- * BroadcastHashJoin Inner BuildRight (42) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- ReusedExchange (27) + : +- BroadcastExchange (34) + : +- * CometColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometNativeScan parquet spark_catalog.default.store (30) + +- BroadcastExchange (41) + +- * CometColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) + +- CometNativeScan parquet spark_catalog.default.date_dim (37) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) -(5) CometBroadcastExchange +(5) CometProject Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7], [d_date_sk#4, d_week_seq#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(d_day_name#6, 9)) AS d_day_name#7] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7] + +(7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(7) CometProject -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] - -(8) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(8) CometProject +Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] -(9) HashAggregate [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(9) CometHashAggregate +Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Results [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] + +(10) CometExchange +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] +Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(10) Exchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(11) CometColumnarToRow [codegen id : 6] +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] -(11) HashAggregate [codegen id : 8] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] +(12) HashAggregate [codegen id : 6] +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END))#15, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END))#16, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END))#17, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END))#18, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END))#19, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))#21] +Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END))#15,17,2) AS sun_sales#22, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END))#16,17,2) AS mon_sales#23, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END))#17,17,2) AS tue_sales#24, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END))#18,17,2) AS wed_sales#25, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END))#19,17,2) AS thu_sales#26, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END))#20,17,2) AS fri_sales#27, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))#21,17,2) AS sat_sales#28] -(12) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Arguments: [s_store_sk#35, s_store_id#36, s_store_name#37] +(13) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#29, s_store_id#30, s_store_name#31] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct -(13) CometFilter -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) +(14) CometFilter +Input [3]: [s_store_sk#29, s_store_id#30, s_store_name#31] +Condition : (isnotnull(s_store_sk#29) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#30, 16)))) -(14) ColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] +(15) CometProject +Input [3]: [s_store_sk#29, s_store_id#30, s_store_name#31] +Arguments: [s_store_sk#29, s_store_id#32, s_store_name#31], [s_store_sk#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#30, 16)) AS s_store_id#32, s_store_name#31] -(15) BroadcastExchange -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(16) CometColumnarToRow [codegen id : 1] +Input [3]: [s_store_sk#29, s_store_id#32, s_store_name#31] -(16) BroadcastHashJoin [codegen id : 8] +(17) BroadcastExchange +Input [3]: [s_store_sk#29, s_store_id#32, s_store_name#31] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(18) BroadcastHashJoin [codegen id : 6] Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#35] +Right keys [1]: [s_store_sk#29] Join type: Inner Join condition: None -(17) Project [codegen id : 8] -Output [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37] -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#35, s_store_id#36, s_store_name#37] +(19) Project [codegen id : 6] +Output [10]: [d_week_seq#5, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, s_store_id#32, s_store_name#31] +Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, s_store_sk#29, s_store_id#32, s_store_name#31] -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#38, d_week_seq#39] -Arguments: [d_month_seq#38, d_week_seq#39] +(20) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#33, d_week_seq#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] +ReadSchema: struct -(19) CometFilter -Input [2]: [d_month_seq#38, d_week_seq#39] -Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= 1212)) AND (d_month_seq#38 <= 1223)) AND isnotnull(d_week_seq#39)) +(21) CometFilter +Input [2]: [d_month_seq#33, d_week_seq#34] +Condition : (((isnotnull(d_month_seq#33) AND (d_month_seq#33 >= 1212)) AND (d_month_seq#33 <= 1223)) AND isnotnull(d_week_seq#34)) -(20) CometProject -Input [2]: [d_month_seq#38, d_week_seq#39] -Arguments: [d_week_seq#39], [d_week_seq#39] +(22) CometProject +Input [2]: [d_month_seq#33, d_week_seq#34] +Arguments: [d_week_seq#34], [d_week_seq#34] -(21) ColumnarToRow [codegen id : 3] -Input [1]: [d_week_seq#39] +(23) CometColumnarToRow [codegen id : 2] +Input [1]: [d_week_seq#34] -(22) BroadcastExchange -Input [1]: [d_week_seq#39] +(24) BroadcastExchange +Input [1]: [d_week_seq#34] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(23) BroadcastHashJoin [codegen id : 8] +(25) BroadcastHashJoin [codegen id : 6] Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#39] +Right keys [1]: [d_week_seq#34] Join type: Inner Join condition: None -(24) Project [codegen id : 8] -Output [10]: [s_store_name#37 AS s_store_name1#40, d_week_seq#5 AS d_week_seq1#41, s_store_id#36 AS s_store_id1#42, sun_sales#28 AS sun_sales1#43, mon_sales#29 AS mon_sales1#44, tue_sales#30 AS tue_sales1#45, wed_sales#31 AS wed_sales1#46, thu_sales#32 AS thu_sales1#47, fri_sales#33 AS fri_sales1#48, sat_sales#34 AS sat_sales1#49] -Input [11]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37, d_week_seq#39] +(26) Project [codegen id : 6] +Output [10]: [s_store_name#31 AS s_store_name1#35, d_week_seq#5 AS d_week_seq1#36, s_store_id#32 AS s_store_id1#37, sun_sales#22 AS sun_sales1#38, mon_sales#23 AS mon_sales1#39, tue_sales#24 AS tue_sales1#40, wed_sales#25 AS wed_sales1#41, thu_sales#26 AS thu_sales1#42, fri_sales#27 AS fri_sales1#43, sat_sales#28 AS sat_sales1#44] +Input [11]: [d_week_seq#5, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, s_store_id#32, s_store_name#31, d_week_seq#34] + +(27) ReusedExchange [Reuses operator id: 10] +Output [9]: [d_week_seq#45, ss_store_sk#46, sum#47, sum#48, sum#49, sum#50, sum#51, sum#52, sum#53] + +(28) CometColumnarToRow [codegen id : 5] +Input [9]: [d_week_seq#45, ss_store_sk#46, sum#47, sum#48, sum#49, sum#50, sum#51, sum#52, sum#53] -(25) ReusedExchange [Reuses operator id: 10] -Output [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] +(29) HashAggregate [codegen id : 5] +Input [9]: [d_week_seq#45, ss_store_sk#46, sum#47, sum#48, sum#49, sum#50, sum#51, sum#52, sum#53] +Keys [2]: [d_week_seq#45, ss_store_sk#46] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#54 = Sunday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Monday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Tuesday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Wednesday) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Thursday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Friday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Saturday ) THEN ss_sales_price#55 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#54 = Sunday ) THEN ss_sales_price#55 END))#15, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Monday ) THEN ss_sales_price#55 END))#16, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Tuesday ) THEN ss_sales_price#55 END))#17, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Wednesday) THEN ss_sales_price#55 END))#18, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Thursday ) THEN ss_sales_price#55 END))#19, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Friday ) THEN ss_sales_price#55 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Saturday ) THEN ss_sales_price#55 END))#21] +Results [9]: [d_week_seq#45, ss_store_sk#46, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Sunday ) THEN ss_sales_price#55 END))#15,17,2) AS sun_sales#56, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Monday ) THEN ss_sales_price#55 END))#16,17,2) AS mon_sales#57, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Tuesday ) THEN ss_sales_price#55 END))#17,17,2) AS tue_sales#58, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Wednesday) THEN ss_sales_price#55 END))#18,17,2) AS wed_sales#59, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Thursday ) THEN ss_sales_price#55 END))#19,17,2) AS thu_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Friday ) THEN ss_sales_price#55 END))#20,17,2) AS fri_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Saturday ) THEN ss_sales_price#55 END))#21,17,2) AS sat_sales#62] -(26) HashAggregate [codegen id : 7] -Input [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] -Keys [2]: [d_week_seq#50, ss_store_sk#51] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27] -Results [9]: [d_week_seq#50, ss_store_sk#51, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21,17,2) AS sun_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22,17,2) AS mon_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23,17,2) AS tue_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24,17,2) AS wed_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25,17,2) AS thu_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26,17,2) AS fri_sales#66, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27,17,2) AS sat_sales#67] +(30) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#63, s_store_id#64] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct -(27) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#68, s_store_id#69] -Arguments: [s_store_sk#68, s_store_id#69] +(31) CometFilter +Input [2]: [s_store_sk#63, s_store_id#64] +Condition : (isnotnull(s_store_sk#63) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#64, 16)))) -(28) CometFilter -Input [2]: [s_store_sk#68, s_store_id#69] -Condition : (isnotnull(s_store_sk#68) AND isnotnull(s_store_id#69)) +(32) CometProject +Input [2]: [s_store_sk#63, s_store_id#64] +Arguments: [s_store_sk#63, s_store_id#65], [s_store_sk#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#64, 16)) AS s_store_id#65] -(29) ColumnarToRow [codegen id : 5] -Input [2]: [s_store_sk#68, s_store_id#69] +(33) CometColumnarToRow [codegen id : 3] +Input [2]: [s_store_sk#63, s_store_id#65] -(30) BroadcastExchange -Input [2]: [s_store_sk#68, s_store_id#69] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(34) BroadcastExchange +Input [2]: [s_store_sk#63, s_store_id#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(31) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_store_sk#51] -Right keys [1]: [s_store_sk#68] +(35) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ss_store_sk#46] +Right keys [1]: [s_store_sk#63] Join type: Inner Join condition: None -(32) Project [codegen id : 7] -Output [9]: [d_week_seq#50, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_id#69] -Input [11]: [d_week_seq#50, ss_store_sk#51, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_sk#68, s_store_id#69] +(36) Project [codegen id : 5] +Output [9]: [d_week_seq#45, sun_sales#56, mon_sales#57, tue_sales#58, wed_sales#59, thu_sales#60, fri_sales#61, sat_sales#62, s_store_id#65] +Input [11]: [d_week_seq#45, ss_store_sk#46, sun_sales#56, mon_sales#57, tue_sales#58, wed_sales#59, thu_sales#60, fri_sales#61, sat_sales#62, s_store_sk#63, s_store_id#65] -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#70, d_week_seq#71] -Arguments: [d_month_seq#70, d_week_seq#71] +(37) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#66, d_week_seq#67] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] +ReadSchema: struct -(34) CometFilter -Input [2]: [d_month_seq#70, d_week_seq#71] -Condition : (((isnotnull(d_month_seq#70) AND (d_month_seq#70 >= 1224)) AND (d_month_seq#70 <= 1235)) AND isnotnull(d_week_seq#71)) +(38) CometFilter +Input [2]: [d_month_seq#66, d_week_seq#67] +Condition : (((isnotnull(d_month_seq#66) AND (d_month_seq#66 >= 1224)) AND (d_month_seq#66 <= 1235)) AND isnotnull(d_week_seq#67)) -(35) CometProject -Input [2]: [d_month_seq#70, d_week_seq#71] -Arguments: [d_week_seq#71], [d_week_seq#71] +(39) CometProject +Input [2]: [d_month_seq#66, d_week_seq#67] +Arguments: [d_week_seq#67], [d_week_seq#67] -(36) ColumnarToRow [codegen id : 6] -Input [1]: [d_week_seq#71] +(40) CometColumnarToRow [codegen id : 4] +Input [1]: [d_week_seq#67] -(37) BroadcastExchange -Input [1]: [d_week_seq#71] +(41) BroadcastExchange +Input [1]: [d_week_seq#67] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -(38) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [d_week_seq#50] -Right keys [1]: [d_week_seq#71] +(42) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [d_week_seq#45] +Right keys [1]: [d_week_seq#67] Join type: Inner Join condition: None -(39) Project [codegen id : 7] -Output [9]: [d_week_seq#50 AS d_week_seq2#72, s_store_id#69 AS s_store_id2#73, sun_sales#61 AS sun_sales2#74, mon_sales#62 AS mon_sales2#75, tue_sales#63 AS tue_sales2#76, wed_sales#64 AS wed_sales2#77, thu_sales#65 AS thu_sales2#78, fri_sales#66 AS fri_sales2#79, sat_sales#67 AS sat_sales2#80] -Input [10]: [d_week_seq#50, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_id#69, d_week_seq#71] +(43) Project [codegen id : 5] +Output [9]: [d_week_seq#45 AS d_week_seq2#68, s_store_id#65 AS s_store_id2#69, sun_sales#56 AS sun_sales2#70, mon_sales#57 AS mon_sales2#71, tue_sales#58 AS tue_sales2#72, wed_sales#59 AS wed_sales2#73, thu_sales#60 AS thu_sales2#74, fri_sales#61 AS fri_sales2#75, sat_sales#62 AS sat_sales2#76] +Input [10]: [d_week_seq#45, sun_sales#56, mon_sales#57, tue_sales#58, wed_sales#59, thu_sales#60, fri_sales#61, sat_sales#62, s_store_id#65, d_week_seq#67] -(40) BroadcastExchange -Input [9]: [d_week_seq2#72, s_store_id2#73, sun_sales2#74, mon_sales2#75, tue_sales2#76, wed_sales2#77, thu_sales2#78, fri_sales2#79, sat_sales2#80] +(44) BroadcastExchange +Input [9]: [d_week_seq2#68, s_store_id2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [plan_id=6] -(41) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [s_store_id1#42, d_week_seq1#41] -Right keys [2]: [s_store_id2#73, (d_week_seq2#72 - 52)] +(45) BroadcastHashJoin [codegen id : 6] +Left keys [2]: [s_store_id1#37, d_week_seq1#36] +Right keys [2]: [s_store_id2#69, (d_week_seq2#68 - 52)] Join type: Inner Join condition: None -(42) Project [codegen id : 8] -Output [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1#43 / sun_sales2#74) AS (sun_sales1 / sun_sales2)#81, (mon_sales1#44 / mon_sales2#75) AS (mon_sales1 / mon_sales2)#82, (tue_sales1#45 / tue_sales2#76) AS (tue_sales1 / tue_sales2)#83, (wed_sales1#46 / wed_sales2#77) AS (wed_sales1 / wed_sales2)#84, (thu_sales1#47 / thu_sales2#78) AS (thu_sales1 / thu_sales2)#85, (fri_sales1#48 / fri_sales2#79) AS (fri_sales1 / fri_sales2)#86, (sat_sales1#49 / sat_sales2#80) AS (sat_sales1 / sat_sales2)#87] -Input [19]: [s_store_name1#40, d_week_seq1#41, s_store_id1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#72, s_store_id2#73, sun_sales2#74, mon_sales2#75, tue_sales2#76, wed_sales2#77, thu_sales2#78, fri_sales2#79, sat_sales2#80] +(46) Project [codegen id : 6] +Output [10]: [s_store_name1#35, s_store_id1#37, d_week_seq1#36, (sun_sales1#38 / sun_sales2#70) AS (sun_sales1 / sun_sales2)#77, (mon_sales1#39 / mon_sales2#71) AS (mon_sales1 / mon_sales2)#78, (tue_sales1#40 / tue_sales2#72) AS (tue_sales1 / tue_sales2)#79, (wed_sales1#41 / wed_sales2#73) AS (wed_sales1 / wed_sales2)#80, (thu_sales1#42 / thu_sales2#74) AS (thu_sales1 / thu_sales2)#81, (fri_sales1#43 / fri_sales2#75) AS (fri_sales1 / fri_sales2)#82, (sat_sales1#44 / sat_sales2#76) AS (sat_sales1 / sat_sales2)#83] +Input [19]: [s_store_name1#35, d_week_seq1#36, s_store_id1#37, sun_sales1#38, mon_sales1#39, tue_sales1#40, wed_sales1#41, thu_sales1#42, fri_sales1#43, sat_sales1#44, d_week_seq2#68, s_store_id2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] -(43) TakeOrderedAndProject -Input [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#81, (mon_sales1 / mon_sales2)#82, (tue_sales1 / tue_sales2)#83, (wed_sales1 / wed_sales2)#84, (thu_sales1 / thu_sales2)#85, (fri_sales1 / fri_sales2)#86, (sat_sales1 / sat_sales2)#87] -Arguments: 100, [s_store_name1#40 ASC NULLS FIRST, s_store_id1#42 ASC NULLS FIRST, d_week_seq1#41 ASC NULLS FIRST], [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#81, (mon_sales1 / mon_sales2)#82, (tue_sales1 / tue_sales2)#83, (wed_sales1 / wed_sales2)#84, (thu_sales1 / thu_sales2)#85, (fri_sales1 / fri_sales2)#86, (sat_sales1 / sat_sales2)#87] +(47) TakeOrderedAndProject +Input [10]: [s_store_name1#35, s_store_id1#37, d_week_seq1#36, (sun_sales1 / sun_sales2)#77, (mon_sales1 / mon_sales2)#78, (tue_sales1 / tue_sales2)#79, (wed_sales1 / wed_sales2)#80, (thu_sales1 / thu_sales2)#81, (fri_sales1 / fri_sales2)#82, (sat_sales1 / sat_sales2)#83] +Arguments: 100, [s_store_name1#35 ASC NULLS FIRST, s_store_id1#37 ASC NULLS FIRST, d_week_seq1#36 ASC NULLS FIRST], [s_store_name1#35, s_store_id1#37, d_week_seq1#36, (sun_sales1 / sun_sales2)#77, (mon_sales1 / mon_sales2)#78, (tue_sales1 / tue_sales2)#79, (wed_sales1 / wed_sales2)#80, (thu_sales1 / thu_sales2)#81, (fri_sales1 / fri_sales2)#82, (sat_sales1 / sat_sales2)#83] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/extended.txt new file mode 100644 index 0000000000..a17ebcc840 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/extended.txt @@ -0,0 +1,58 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_week_seq#1, ss_store_sk#2, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#4 END))#5,17,2) AS sun_sales#6, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#4 END))#7,17,2) AS mon_sales#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#4 END))#9,17,2) AS tue_sales#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#4 END))#11,17,2) AS wed_sales#12, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#4 END))#13,17,2) AS thu_sales#14, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#4 END))#15,17,2) AS fri_sales#16, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#4 END))#17,17,2) AS sat_sales#18)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_week_seq#19, ss_store_sk#20, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#21 = Sunday ) THEN ss_sales_price#22 END))#5,17,2) AS sun_sales#23, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#21 = Monday ) THEN ss_sales_price#22 END))#7,17,2) AS mon_sales#24, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#21 = Tuesday ) THEN ss_sales_price#22 END))#9,17,2) AS tue_sales#25, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#21 = Wednesday) THEN ss_sales_price#22 END))#11,17,2) AS wed_sales#26, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#21 = Thursday ) THEN ss_sales_price#22 END))#13,17,2) AS thu_sales#27, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#21 = Friday ) THEN ss_sales_price#22 END))#15,17,2) AS fri_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#21 = Saturday ) THEN ss_sales_price#22 END))#17,17,2) AS sat_sales#29)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 50 eligible operators (64%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/simplified.txt index bb9542ccb7..78d230ab69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] - WholeStageCodegen (8) + WholeStageCodegen (6) Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2] Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] @@ -7,56 +7,58 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] BroadcastHashJoin [ss_store_sk,s_store_sk] HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq,ss_store_sk] #1 - WholeStageCodegen (1) - HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] + CometColumnarToRow + InputAdapter + CometExchange [d_week_seq,ss_store_sk] #1 + CometHashAggregate [d_day_name,ss_sales_price] [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] + CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #3 - WholeStageCodegen (2) - ColumnarToRow + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] InputAdapter BroadcastExchange #4 - WholeStageCodegen (3) - ColumnarToRow + WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] InputAdapter BroadcastExchange #5 - WholeStageCodegen (7) + WholeStageCodegen (5) Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] BroadcastHashJoin [ss_store_sk,s_store_sk] HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 + CometColumnarToRow + InputAdapter + ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 InputAdapter BroadcastExchange #6 - WholeStageCodegen (5) - ColumnarToRow + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] InputAdapter BroadcastExchange #7 - WholeStageCodegen (6) - ColumnarToRow + WholeStageCodegen (4) + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/explain.txt index 7ed058c86b..14ccdb731f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/explain.txt @@ -1,67 +1,89 @@ == Physical Plan == -TakeOrderedAndProject (50) -+- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- Union (46) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometProject (33) - : : : +- CometBroadcastHashJoin (32) - : : : :- CometFilter (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : : : +- ReusedExchange (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- * HashAggregate (45) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (65) ++- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometColumnarExchange (62) + +- * HashAggregate (61) + +- Union (60) + :- * HashAggregate (29) + : +- * CometColumnarToRow (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- * HashAggregate (44) + : +- * CometColumnarToRow (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (34) + : : : +- CometBroadcastHashJoin (33) + : : : :- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (30) + : : : +- ReusedExchange (32) + : : +- ReusedExchange (35) + : +- ReusedExchange (38) + +- * HashAggregate (59) + +- * CometColumnarToRow (58) + +- CometExchange (57) + +- CometHashAggregate (56) + +- CometProject (55) + +- CometBroadcastHashJoin (54) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (49) + : : +- CometBroadcastHashJoin (48) + : : :- CometFilter (46) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (45) + : : +- ReusedExchange (47) + : +- ReusedExchange (50) + +- ReusedExchange (53) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] @@ -84,9 +106,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] @@ -109,165 +134,235 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometNativeScan: `spark_catalog`.`default`.`item` +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12, i_category#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category)] +ReadSchema: struct (18) CometFilter Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) +Condition : (isnotnull(i_category#13) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#13, 50)) = Music )) (19) CometProject Input [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12], [i_item_id#12] +Arguments: [i_item_id#14], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight +Right output [1]: [i_item_id#14] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16))], [i_item_id#14], LeftSemi, BuildRight -(22) CometBroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16)) AS i_item_id#15] -(23) CometBroadcastHashJoin +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] +Right output [2]: [i_item_sk#10, i_item_id#15] Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] - -(25) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(26) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] -(27) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] -(28) HashAggregate [codegen id : 2] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] +(29) HashAggregate [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#22] - -(32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] - -(34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#23] - -(35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#23] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#23], Inner, BuildRight - -(36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] - -(37) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#24, i_item_id#25] - -(38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#24, i_item_id#25] -Arguments: [cs_item_sk#19], [i_item_sk#24], Inner, BuildRight - -(39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] -Arguments: [cs_ext_sales_price#20, i_item_id#25], [cs_ext_sales_price#20, i_item_id#25] - -(40) ColumnarToRow [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] - -(41) HashAggregate [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(42) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(43) HashAggregate [codegen id : 4] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(44) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_id#30, sum#31] - -(45) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#30, sum#31] -Keys [1]: [i_item_id#30] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#33] -Results [2]: [i_item_id#30, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#33,17,2) AS total_sales#34] - -(46) Union - -(47) HashAggregate [codegen id : 7] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [i_item_id#11, sum#37, isEmpty#38] - -(48) Exchange -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(49) HashAggregate [codegen id : 8] -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#39] -Results [2]: [i_item_id#11, sum(total_sales#17)#39 AS total_sales#40] - -(50) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#40] -Arguments: 100, [i_item_id#11 ASC NULLS FIRST, total_sales#40 ASC NULLS FIRST], [i_item_id#11, total_sales#40] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] +Results [2]: [i_item_id#15 AS i_item_id#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#19] + +(30) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#23)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Condition : (isnotnull(cs_bill_addr_sk#20) AND isnotnull(cs_item_sk#21)) + +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#24] + +(33) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Right output [1]: [d_date_sk#24] +Arguments: [cs_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight + +(34) CometProject +Input [5]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, d_date_sk#24] +Arguments: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22], [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] + +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#25] + +(36) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] +Right output [1]: [ca_address_sk#25] +Arguments: [cs_bill_addr_sk#20], [ca_address_sk#25], Inner, BuildRight + +(37) CometProject +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, ca_address_sk#25] +Arguments: [cs_item_sk#21, cs_ext_sales_price#22], [cs_item_sk#21, cs_ext_sales_price#22] + +(38) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#26, i_item_id#27] + +(39) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#21, cs_ext_sales_price#22] +Right output [2]: [i_item_sk#26, i_item_id#27] +Arguments: [cs_item_sk#21], [i_item_sk#26], Inner, BuildRight + +(40) CometProject +Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, i_item_sk#26, i_item_id#27] +Arguments: [cs_ext_sales_price#22, i_item_id#27], [cs_ext_sales_price#22, i_item_id#27] + +(41) CometHashAggregate +Input [2]: [cs_ext_sales_price#22, i_item_id#27] +Keys [1]: [i_item_id#27] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))] + +(42) CometExchange +Input [2]: [i_item_id#27, sum#28] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(43) CometColumnarToRow [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] + +(44) HashAggregate [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] +Keys [1]: [i_item_id#27] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#29] +Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#29,17,2) AS total_sales#30] + +(45) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(46) CometFilter +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : (isnotnull(ws_bill_addr_sk#32) AND isnotnull(ws_item_sk#31)) + +(47) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#35] + +(48) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [1]: [d_date_sk#35] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight + +(49) CometProject +Input [5]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35] +Arguments: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33], [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] + +(50) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#36] + +(51) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] +Right output [1]: [ca_address_sk#36] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#36], Inner, BuildRight + +(52) CometProject +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ca_address_sk#36] +Arguments: [ws_item_sk#31, ws_ext_sales_price#33], [ws_item_sk#31, ws_ext_sales_price#33] + +(53) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#37, i_item_id#38] + +(54) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#31, ws_ext_sales_price#33] +Right output [2]: [i_item_sk#37, i_item_id#38] +Arguments: [ws_item_sk#31], [i_item_sk#37], Inner, BuildRight + +(55) CometProject +Input [4]: [ws_item_sk#31, ws_ext_sales_price#33, i_item_sk#37, i_item_id#38] +Arguments: [ws_ext_sales_price#33, i_item_id#38], [ws_ext_sales_price#33, i_item_id#38] + +(56) CometHashAggregate +Input [2]: [ws_ext_sales_price#33, i_item_id#38] +Keys [1]: [i_item_id#38] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] + +(57) CometExchange +Input [2]: [i_item_id#38, sum#39] +Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(58) CometColumnarToRow [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] + +(59) HashAggregate [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] +Keys [1]: [i_item_id#38] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#40] +Results [2]: [i_item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#40,17,2) AS total_sales#41] + +(60) Union + +(61) HashAggregate [codegen id : 4] +Input [2]: [i_item_id#18, total_sales#19] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(total_sales#19)] +Aggregate Attributes [2]: [sum#42, isEmpty#43] +Results [3]: [i_item_id#18, sum#44, isEmpty#45] + +(62) CometColumnarExchange +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(63) CometColumnarToRow [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] + +(64) HashAggregate [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(total_sales#19)] +Aggregate Attributes [1]: [sum(total_sales#19)#46] +Results [2]: [i_item_id#18, sum(total_sales#19)#46 AS total_sales#47] + +(65) TakeOrderedAndProject +Input [2]: [i_item_id#18, total_sales#47] +Arguments: 100, [i_item_id#18 ASC NULLS FIRST, total_sales#47 ASC NULLS FIRST], [i_item_id#18, total_sales#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/extended.txt new file mode 100644 index 0000000000..a65cb49e1b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/extended.txt @@ -0,0 +1,95 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#1 AS i_item_id#2, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#4,17,2) AS total_sales#5)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#6, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#7))#8,17,2) AS total_sales#9)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#10, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#11))#12,17,2) AS total_sales#13)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 82 out of 89 eligible operators (92%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/simplified.txt index 00efd9736a..c754a27bd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/simplified.txt @@ -1,64 +1,75 @@ TakeOrderedAndProject [i_item_id,total_sales] - WholeStageCodegen (8) + WholeStageCodegen (5) HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_item_id] #1 + WholeStageCodegen (4) + HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),i_item_id,total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] + CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_category] - WholeStageCodegen (4) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (3) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - ReusedExchange [i_item_id,sum] #2 + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_category] + WholeStageCodegen (2) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + WholeStageCodegen (3) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/explain.txt index 396c7ee90d..e7580eeb9e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/explain.txt @@ -1,14 +1,14 @@ == Physical Plan == -* Project (62) -+- * BroadcastNestedLoopJoin Inner BuildRight (61) - :- * HashAggregate (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- * ColumnarToRow (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) +* Project (65) ++- * BroadcastNestedLoopJoin Inner BuildRight (64) + :- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) : : :- CometProject (25) : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) @@ -18,62 +18,72 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (15) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (21) - : : +- ReusedExchange (26) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (29) - +- BroadcastExchange (60) - +- * HashAggregate (59) - +- Exchange (58) - +- * HashAggregate (57) - +- * ColumnarToRow (56) - +- CometProject (55) - +- CometBroadcastHashJoin (54) - :- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (49) - : : +- CometBroadcastHashJoin (48) - : : :- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometFilter (40) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (39) - : : : : +- ReusedExchange (41) - : : : +- ReusedExchange (44) - : : +- ReusedExchange (47) - : +- ReusedExchange (50) - +- ReusedExchange (53) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.customer (21) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.item (32) + +- BroadcastExchange (63) + +- * HashAggregate (62) + +- * CometColumnarToRow (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometProject (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometProject (49) + : : : +- CometBroadcastHashJoin (48) + : : : :- CometProject (46) + : : : : +- CometBroadcastHashJoin (45) + : : : : :- CometFilter (43) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (42) + : : : : +- ReusedExchange (44) + : : : +- ReusedExchange (47) + : : +- ReusedExchange (50) + : +- ReusedExchange (53) + +- ReusedExchange (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`store` +(3) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#7, s_gmt_offset#8] -Arguments: [s_store_sk#7, s_gmt_offset#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [s_store_sk#7, s_gmt_offset#8] @@ -96,13 +106,16 @@ Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) CometNativeScan: `spark_catalog`.`default`.`promotion` +(9) CometNativeScan parquet spark_catalog.default.promotion Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Arguments: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct (10) CometFilter Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_dmail#10, 1)) = Y) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#11, 1)) = Y)) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#12, 1)) = Y)) AND isnotnull(p_promo_sk#9)) (11) CometProject Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] @@ -121,9 +134,12 @@ Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -(15) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(15) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_year#14, d_moy#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] @@ -146,9 +162,12 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -(21) CometNativeScan: `spark_catalog`.`default`.`customer` +(21) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#16, c_current_addr_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (22) CometFilter Input [2]: [c_customer_sk#16, c_current_addr_sk#17] @@ -167,162 +186,181 @@ Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -(26) ReusedExchange [Reuses operator id: 6] -Output [1]: [ca_address_sk#18] +(26) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#18, ca_gmt_offset#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) + +(28) CometProject +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] + +(29) CometBroadcastExchange +Input [1]: [ca_address_sk#18] +Arguments: [ca_address_sk#18] -(27) CometBroadcastHashJoin +(30) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] Right output [1]: [ca_address_sk#18] Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight -(28) CometProject +(31) CometProject Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] -(29) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19, i_category#20] +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#20, i_category#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk)] +ReadSchema: struct -(30) CometFilter -Input [2]: [i_item_sk#19, i_category#20] -Condition : ((isnotnull(i_category#20) AND (i_category#20 = Jewelry )) AND isnotnull(i_item_sk#19)) +(33) CometFilter +Input [2]: [i_item_sk#20, i_category#21] +Condition : ((isnotnull(i_category#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#21, 50)) = Jewelry )) AND isnotnull(i_item_sk#20)) -(31) CometProject -Input [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +(34) CometProject +Input [2]: [i_item_sk#20, i_category#21] +Arguments: [i_item_sk#20], [i_item_sk#20] -(32) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +(35) CometBroadcastExchange +Input [1]: [i_item_sk#20] +Arguments: [i_item_sk#20] -(33) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight -(34) CometProject -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#19] +(37) CometProject +Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] -(35) ColumnarToRow [codegen id : 1] -Input [1]: [ss_ext_sales_price#5] - -(36) HashAggregate [codegen id : 1] +(38) CometHashAggregate Input [1]: [ss_ext_sales_price#5] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#21] -Results [1]: [sum#22] -(37) Exchange +(39) CometExchange +Input [1]: [sum#22] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometColumnarToRow [codegen id : 2] Input [1]: [sum#22] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] -(38) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 2] Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#23] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#23,17,2) AS promotions#24] -(39) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(42) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#29)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct -(40) CometFilter +(43) CometFilter Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) -(41) ReusedExchange [Reuses operator id: 6] +(44) ReusedExchange [Reuses operator id: 6] Output [1]: [s_store_sk#30] -(42) CometBroadcastHashJoin +(45) CometBroadcastHashJoin Left output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] Right output [1]: [s_store_sk#30] Arguments: [ss_store_sk#27], [s_store_sk#30], Inner, BuildRight -(43) CometProject +(46) CometProject Input [6]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, s_store_sk#30] Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] -(44) ReusedExchange [Reuses operator id: 18] +(47) ReusedExchange [Reuses operator id: 18] Output [1]: [d_date_sk#31] -(45) CometBroadcastHashJoin +(48) CometBroadcastHashJoin Left output [4]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] Right output [1]: [d_date_sk#31] Arguments: [ss_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight -(46) CometProject +(49) CometProject Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#31] Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] -(47) ReusedExchange [Reuses operator id: 23] +(50) ReusedExchange [Reuses operator id: 23] Output [2]: [c_customer_sk#32, c_current_addr_sk#33] -(48) CometBroadcastHashJoin +(51) CometBroadcastHashJoin Left output [3]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] Right output [2]: [c_customer_sk#32, c_current_addr_sk#33] Arguments: [ss_customer_sk#26], [c_customer_sk#32], Inner, BuildRight -(49) CometProject +(52) CometProject Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, c_customer_sk#32, c_current_addr_sk#33] Arguments: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33], [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] -(50) ReusedExchange [Reuses operator id: 6] +(53) ReusedExchange [Reuses operator id: 29] Output [1]: [ca_address_sk#34] -(51) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [3]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] Right output [1]: [ca_address_sk#34] Arguments: [c_current_addr_sk#33], [ca_address_sk#34], Inner, BuildRight -(52) CometProject +(55) CometProject Input [4]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33, ca_address_sk#34] Arguments: [ss_item_sk#25, ss_ext_sales_price#28], [ss_item_sk#25, ss_ext_sales_price#28] -(53) ReusedExchange [Reuses operator id: 32] +(56) ReusedExchange [Reuses operator id: 35] Output [1]: [i_item_sk#35] -(54) CometBroadcastHashJoin +(57) CometBroadcastHashJoin Left output [2]: [ss_item_sk#25, ss_ext_sales_price#28] Right output [1]: [i_item_sk#35] Arguments: [ss_item_sk#25], [i_item_sk#35], Inner, BuildRight -(55) CometProject +(58) CometProject Input [3]: [ss_item_sk#25, ss_ext_sales_price#28, i_item_sk#35] Arguments: [ss_ext_sales_price#28], [ss_ext_sales_price#28] -(56) ColumnarToRow [codegen id : 2] -Input [1]: [ss_ext_sales_price#28] - -(57) HashAggregate [codegen id : 2] +(59) CometHashAggregate Input [1]: [ss_ext_sales_price#28] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] -Aggregate Attributes [1]: [sum#36] -Results [1]: [sum#37] -(58) Exchange -Input [1]: [sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(60) CometExchange +Input [1]: [sum#36] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(61) CometColumnarToRow [codegen id : 1] +Input [1]: [sum#36] -(59) HashAggregate [codegen id : 3] -Input [1]: [sum#37] +(62) HashAggregate [codegen id : 1] +Input [1]: [sum#36] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#28))#38] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#28))#38,17,2) AS total#39] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#28))#37] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#28))#37,17,2) AS total#38] -(60) BroadcastExchange -Input [1]: [total#39] +(63) BroadcastExchange +Input [1]: [total#38] Arguments: IdentityBroadcastMode, [plan_id=3] -(61) BroadcastNestedLoopJoin [codegen id : 4] +(64) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(62) Project [codegen id : 4] -Output [3]: [promotions#24, total#39, ((cast(promotions#24 as decimal(15,4)) / cast(total#39 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#40] -Input [2]: [promotions#24, total#39] +(65) Project [codegen id : 2] +Output [3]: [promotions#24, total#38, ((cast(promotions#24 as decimal(15,4)) / cast(total#38 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#39] +Input [2]: [promotions#24, total#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/extended.txt new file mode 100644 index 0000000000..727d324f2a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/extended.txt @@ -0,0 +1,81 @@ +Project ++- BroadcastNestedLoopJoin + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#1))#2,17,2) AS promotions#3)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#4))#5,17,2) AS total#6)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 72 out of 77 eligible operators (93%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/simplified.txt index 158c3657ff..cafea99086 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/simplified.txt @@ -1,71 +1,70 @@ -WholeStageCodegen (4) +WholeStageCodegen (2) Project [promotions,total] BroadcastNestedLoopJoin HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_gmt_offset] - CometBroadcastExchange [p_promo_sk] #3 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - ReusedExchange [ca_address_sk] #2 - CometBroadcastExchange [i_item_sk] #6 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] + CometBroadcastExchange [p_promo_sk] #3 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) + BroadcastExchange #8 + WholeStageCodegen (1) HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] - InputAdapter - Exchange #8 - WholeStageCodegen (2) - HashAggregate [ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedExchange [s_store_sk] #2 - ReusedExchange [d_date_sk] #4 - ReusedExchange [c_customer_sk,c_current_addr_sk] #5 - ReusedExchange [ca_address_sk] #2 - ReusedExchange [i_item_sk] #6 + CometColumnarToRow + InputAdapter + CometExchange #9 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [s_store_sk] #2 + ReusedExchange [d_date_sk] #4 + ReusedExchange [c_customer_sk,c_current_addr_sk] #5 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/explain.txt index a2042e5626..9c710b15fb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/explain.txt @@ -1,41 +1,52 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.web_site (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -54,77 +65,104 @@ Arguments: [ws_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -(8) ReusedExchange [Reuses operator id: 5] +(8) CometNativeScan parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct -(9) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight +(9) CometFilter +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) (10) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_type#9, 30)) AS sm_type#10] -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [web_site_sk#10, web_name#11] +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] (12) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [web_site_sk#10, web_name#11] -Arguments: [ws_web_site_sk#2], [web_site_sk#10], Inner, BuildRight +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight (13) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] +(14) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#11, web_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [ws_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) ColumnarToRow [codegen id : 1] -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] - -(21) HashAggregate [codegen id : 1] -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +Input [2]: [web_site_sk#11, web_name#12] +Condition : isnotnull(web_site_sk#11) + +(16) CometBroadcastExchange +Input [2]: [web_site_sk#11, web_name#12] +Arguments: [web_site_sk#11, web_name#12] + +(17) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [web_site_sk#11, web_name#12] +Arguments: [ws_web_site_sk#2], [web_site_sk#11], Inner, BuildRight + +(18) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_site_sk#11, web_name#12] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] + +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) + +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(23) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight + +(24) CometProject +Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12, d_date_sk#13] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] + +(25) CometHashAggregate +Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(22) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, web_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 2] -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(24) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,web_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,web_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, web_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/extended.txt new file mode 100644 index 0000000000..52d7828b06 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.ship_mode + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/simplified.txt index 0f5b318b16..de80c17349 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/simplified.txt @@ -1,28 +1,31 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (2) - HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,web_name] #1 - WholeStageCodegen (1) - HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] - CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] - CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [web_site_sk,web_name] #2 - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,web_name] #1 + CometHashAggregate [ws_ship_date_sk,ws_sold_date_sk] [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] + CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] + CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [web_site_sk,web_name] #4 + CometFilter [web_site_sk,web_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/explain.txt index 4ee8154ba1..8a26ab6ac3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/explain.txt @@ -1,50 +1,58 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- * Filter (27) - +- Window (26) - +- * Sort (25) - +- Exchange (24) - +- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (30) ++- * Project (29) + +- * Filter (28) + +- Window (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Books ,Children ,Electronics ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (personal ,portable ,refernece ,self-help )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Women ,Music ,Men ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (accessories ,classical ,fragrances ,pants )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) (3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] @@ -63,9 +71,12 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_moy#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] @@ -88,9 +99,12 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [1]: [s_store_sk#17] @@ -109,48 +123,49 @@ Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] -(20) ColumnarToRow [codegen id : 1] -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] - -(21) HashAggregate [codegen id : 1] +(20) CometHashAggregate Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manager_id#5, d_moy#16, sum#19] -(22) Exchange -Input [3]: [i_manager_id#5, d_moy#16, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(21) CometExchange +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] -(23) HashAggregate [codegen id : 2] -Input [3]: [i_manager_id#5, d_moy#16, sum#19] +(23) HashAggregate [codegen id : 1] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#19] +Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS _w0#21] + +(24) CometColumnarExchange +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(24) Exchange -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(25) CometSort +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [i_manager_id#5, sum_sales#20, _w0#21], [i_manager_id#5 ASC NULLS FIRST] -(25) Sort [codegen id : 3] -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 +(26) CometColumnarToRow [codegen id : 2] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] -(26) Window -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#23], [i_manager_id#5] +(27) Window +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] -(27) Filter [codegen id : 4] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] -Condition : CASE WHEN (avg_monthly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_monthly_sales#23)) / avg_monthly_sales#23) > 0.1000000000000000) ELSE false END +(28) Filter [codegen id : 3] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Condition : CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END -(28) Project [codegen id : 4] -Output [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] +(29) Project [codegen id : 3] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] -(29) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST], [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] +(30) TakeOrderedAndProject +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/extended.txt new file mode 100644 index 0000000000..1b296c8a1f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/extended.txt @@ -0,0 +1,32 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_manager_id#1, MakeDecimal(sum(UnscaledValue(ss_sales_price#2))#3,17,2) AS sum_sales#4, MakeDecimal(sum(UnscaledValue(ss_sales_price#2))#3,17,2) AS _w0#5)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 23 out of 28 eligible operators (82%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/simplified.txt index eca9e8c6ee..8365af3e19 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/simplified.txt @@ -1,37 +1,36 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [i_manager_id,sum_sales,avg_monthly_sales] Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_manager_id] - WholeStageCodegen (3) - Sort [i_manager_id] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_manager_id] #1 - WholeStageCodegen (2) - HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manager_id,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_manager_id,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manager_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_moy] #4 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] + CometSort [i_manager_id,sum_sales,_w0] + CometColumnarExchange [i_manager_id] #1 + WholeStageCodegen (1) + HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manager_id,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_manager_id,d_moy,sum] + CometProject [i_manager_id,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manager_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_month_seq,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/explain.txt index b9f587226f..5dd3c6ce59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/explain.txt @@ -1,185 +1,196 @@ == Physical Plan == -* ColumnarToRow (176) -+- CometSort (175) - +- CometColumnarExchange (174) - +- * Project (173) - +- * SortMergeJoin Inner (172) - :- * Sort (109) - : +- Exchange (108) - : +- * HashAggregate (107) - : +- * HashAggregate (106) - : +- * Project (105) - : +- * BroadcastHashJoin Inner BuildRight (104) - : :- * Project (98) - : : +- * BroadcastHashJoin Inner BuildRight (97) - : : :- * Project (95) - : : : +- * BroadcastHashJoin Inner BuildRight (94) - : : : :- * Project (92) - : : : : +- * BroadcastHashJoin Inner BuildRight (91) - : : : : :- * Project (89) - : : : : : +- * BroadcastHashJoin Inner BuildRight (88) - : : : : : :- * Project (83) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (82) - : : : : : : :- * Project (80) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : : : : : :- * Project (74) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (73) - : : : : : : : : :- * Project (68) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (67) - : : : : : : : : : :- * Project (65) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : : : : : : : : :- * Project (59) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : : : : : : : : :- * Project (56) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (55) - : : : : : : : : : : : : :- * Project (50) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (49) - : : : : : : : : : : : : : :- * Project (44) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (43) +* CometColumnarToRow (183) ++- CometSort (182) + +- CometExchange (181) + +- CometProject (180) + +- CometSortMergeJoin (179) + :- CometSort (115) + : +- CometColumnarExchange (114) + : +- * HashAggregate (113) + : +- * HashAggregate (112) + : +- * Project (111) + : +- * BroadcastHashJoin Inner BuildRight (110) + : :- * Project (104) + : : +- * BroadcastHashJoin Inner BuildRight (103) + : : :- * Project (101) + : : : +- * BroadcastHashJoin Inner BuildRight (100) + : : : :- * Project (95) + : : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : : :- * Project (92) + : : : : : +- * BroadcastHashJoin Inner BuildRight (91) + : : : : : :- * Project (85) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) + : : : : : : :- * Project (82) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) + : : : : : : : :- * Project (76) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) + : : : : : : : : :- * Project (70) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) + : : : : : : : : : :- * Project (67) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) + : : : : : : : : : : :- * Project (60) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (59) + : : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : : :- * Project (51) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) + : : : : : : : : : : : : : :- * Project (45) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (44) : : : : : : : : : : : : : : :- * Project (38) : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (37) : : : : : : : : : : : : : : : :- * Project (32) : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) - : : : : : : : : : : : : : : : : :- * ColumnarToRow (11) + : : : : : : : : : : : : : : : : :- * CometColumnarToRow (11) : : : : : : : : : : : : : : : : : +- CometSort (10) : : : : : : : : : : : : : : : : : +- CometExchange (9) : : : : : : : : : : : : : : : : : +- CometProject (8) : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- * Sort (30) : : : : : : : : : : : : : : : : +- * Project (29) : : : : : : : : : : : : : : : : +- * Filter (28) : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- Exchange (26) - : : : : : : : : : : : : : : : : +- * HashAggregate (25) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (24) + : : : : : : : : : : : : : : : : +- * CometColumnarToRow (26) + : : : : : : : : : : : : : : : : +- CometExchange (25) + : : : : : : : : : : : : : : : : +- CometHashAggregate (24) : : : : : : : : : : : : : : : : +- CometProject (23) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) : : : : : : : : : : : : : : : : :- CometSort (16) : : : : : : : : : : : : : : : : : +- CometExchange (15) : : : : : : : : : : : : : : : : : +- CometProject (14) : : : : : : : : : : : : : : : : : +- CometFilter (13) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (12) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (12) : : : : : : : : : : : : : : : : +- CometSort (21) : : : : : : : : : : : : : : : : +- CometExchange (20) : : : : : : : : : : : : : : : : +- CometProject (19) : : : : : : : : : : : : : : : : +- CometFilter (18) - : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (17) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (17) : : : : : : : : : : : : : : : +- BroadcastExchange (36) - : : : : : : : : : : : : : : : +- * ColumnarToRow (35) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (35) : : : : : : : : : : : : : : : +- CometFilter (34) - : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - : : : : : : : : : : : : : : +- BroadcastExchange (42) - : : : : : : : : : : : : : : +- * ColumnarToRow (41) - : : : : : : : : : : : : : : +- CometFilter (40) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (39) - : : : : : : : : : : : : : +- BroadcastExchange (48) - : : : : : : : : : : : : : +- * ColumnarToRow (47) - : : : : : : : : : : : : : +- CometFilter (46) - : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (45) - : : : : : : : : : : : : +- BroadcastExchange (54) - : : : : : : : : : : : : +- * ColumnarToRow (53) - : : : : : : : : : : : : +- CometFilter (52) - : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (51) - : : : : : : : : : : : +- ReusedExchange (57) - : : : : : : : : : : +- BroadcastExchange (63) - : : : : : : : : : : +- * ColumnarToRow (62) - : : : : : : : : : : +- CometFilter (61) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (60) - : : : : : : : : : +- ReusedExchange (66) - : : : : : : : : +- BroadcastExchange (72) - : : : : : : : : +- * ColumnarToRow (71) - : : : : : : : : +- CometFilter (70) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (69) - : : : : : : : +- BroadcastExchange (78) - : : : : : : : +- * ColumnarToRow (77) - : : : : : : : +- CometFilter (76) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (75) - : : : : : : +- ReusedExchange (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- * ColumnarToRow (86) - : : : : : +- CometFilter (85) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (84) - : : : : +- ReusedExchange (90) - : : : +- ReusedExchange (93) - : : +- ReusedExchange (96) - : +- BroadcastExchange (103) - : +- * ColumnarToRow (102) - : +- CometProject (101) - : +- CometFilter (100) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (99) - +- * Sort (171) - +- Exchange (170) - +- * HashAggregate (169) - +- * HashAggregate (168) - +- * Project (167) - +- * BroadcastHashJoin Inner BuildRight (166) - :- * Project (164) - : +- * BroadcastHashJoin Inner BuildRight (163) - : :- * Project (161) - : : +- * BroadcastHashJoin Inner BuildRight (160) - : : :- * Project (158) - : : : +- * BroadcastHashJoin Inner BuildRight (157) - : : : :- * Project (155) - : : : : +- * BroadcastHashJoin Inner BuildRight (154) - : : : : :- * Project (152) - : : : : : +- * BroadcastHashJoin Inner BuildRight (151) - : : : : : :- * Project (149) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (148) - : : : : : : :- * Project (146) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (145) - : : : : : : : :- * Project (143) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (142) - : : : : : : : : :- * Project (140) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (139) - : : : : : : : : : :- * Project (137) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (136) - : : : : : : : : : : :- * Project (134) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (133) - : : : : : : : : : : : :- * Project (131) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (130) - : : : : : : : : : : : : :- * Project (128) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (127) - : : : : : : : : : : : : : :- * Project (125) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (124) - : : : : : : : : : : : : : : :- * Project (119) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (118) - : : : : : : : : : : : : : : : :- * ColumnarToRow (112) - : : : : : : : : : : : : : : : : +- CometSort (111) - : : : : : : : : : : : : : : : : +- ReusedExchange (110) - : : : : : : : : : : : : : : : +- * Sort (117) - : : : : : : : : : : : : : : : +- * Project (116) - : : : : : : : : : : : : : : : +- * Filter (115) - : : : : : : : : : : : : : : : +- * HashAggregate (114) - : : : : : : : : : : : : : : : +- ReusedExchange (113) - : : : : : : : : : : : : : : +- BroadcastExchange (123) - : : : : : : : : : : : : : : +- * ColumnarToRow (122) - : : : : : : : : : : : : : : +- CometFilter (121) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (120) - : : : : : : : : : : : : : +- ReusedExchange (126) - : : : : : : : : : : : : +- ReusedExchange (129) - : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : +- ReusedExchange (141) - : : : : : : : +- ReusedExchange (144) - : : : : : : +- ReusedExchange (147) - : : : : : +- ReusedExchange (150) - : : : : +- ReusedExchange (153) - : : : +- ReusedExchange (156) - : : +- ReusedExchange (159) - : +- ReusedExchange (162) - +- ReusedExchange (165) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (33) + : : : : : : : : : : : : : : +- BroadcastExchange (43) + : : : : : : : : : : : : : : +- * CometColumnarToRow (42) + : : : : : : : : : : : : : : +- CometProject (41) + : : : : : : : : : : : : : : +- CometFilter (40) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (39) + : : : : : : : : : : : : : +- BroadcastExchange (49) + : : : : : : : : : : : : : +- * CometColumnarToRow (48) + : : : : : : : : : : : : : +- CometFilter (47) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (46) + : : : : : : : : : : : : +- BroadcastExchange (55) + : : : : : : : : : : : : +- * CometColumnarToRow (54) + : : : : : : : : : : : : +- CometFilter (53) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (52) + : : : : : : : : : : : +- ReusedExchange (58) + : : : : : : : : : : +- BroadcastExchange (65) + : : : : : : : : : : +- * CometColumnarToRow (64) + : : : : : : : : : : +- CometProject (63) + : : : : : : : : : : +- CometFilter (62) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (61) + : : : : : : : : : +- ReusedExchange (68) + : : : : : : : : +- BroadcastExchange (74) + : : : : : : : : +- * CometColumnarToRow (73) + : : : : : : : : +- CometFilter (72) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (71) + : : : : : : : +- BroadcastExchange (80) + : : : : : : : +- * CometColumnarToRow (79) + : : : : : : : +- CometFilter (78) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (77) + : : : : : : +- ReusedExchange (83) + : : : : : +- BroadcastExchange (90) + : : : : : +- * CometColumnarToRow (89) + : : : : : +- CometProject (88) + : : : : : +- CometFilter (87) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (86) + : : : : +- ReusedExchange (93) + : : : +- BroadcastExchange (99) + : : : +- * CometColumnarToRow (98) + : : : +- CometFilter (97) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (96) + : : +- ReusedExchange (102) + : +- BroadcastExchange (109) + : +- * CometColumnarToRow (108) + : +- CometProject (107) + : +- CometFilter (106) + : +- CometNativeScan parquet spark_catalog.default.item (105) + +- CometSort (178) + +- CometColumnarExchange (177) + +- * HashAggregate (176) + +- * HashAggregate (175) + +- * Project (174) + +- * BroadcastHashJoin Inner BuildRight (173) + :- * Project (171) + : +- * BroadcastHashJoin Inner BuildRight (170) + : :- * Project (168) + : : +- * BroadcastHashJoin Inner BuildRight (167) + : : :- * Project (165) + : : : +- * BroadcastHashJoin Inner BuildRight (164) + : : : :- * Project (162) + : : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : : :- * Project (159) + : : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : : :- * Project (156) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : : :- * Project (153) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : : :- * Project (150) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : : :- * Project (147) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : : :- * Project (144) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : : :- * Project (141) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : : :- * Project (126) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (125) + : : : : : : : : : : : : : : : :- * CometColumnarToRow (118) + : : : : : : : : : : : : : : : : +- CometSort (117) + : : : : : : : : : : : : : : : : +- ReusedExchange (116) + : : : : : : : : : : : : : : : +- * Sort (124) + : : : : : : : : : : : : : : : +- * Project (123) + : : : : : : : : : : : : : : : +- * Filter (122) + : : : : : : : : : : : : : : : +- * HashAggregate (121) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- BroadcastExchange (130) + : : : : : : : : : : : : : : +- * CometColumnarToRow (129) + : : : : : : : : : : : : : : +- CometFilter (128) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (127) + : : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : : +- ReusedExchange (145) + : : : : : : : : +- ReusedExchange (148) + : : : : : : : +- ReusedExchange (151) + : : : : : : +- ReusedExchange (154) + : : : : : +- ReusedExchange (157) + : : : : +- ReusedExchange (160) + : : : +- ReusedExchange (163) + : : +- ReusedExchange (166) + : +- ReusedExchange (169) + +- ReusedExchange (172) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] +ReadSchema: struct (2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -189,9 +200,12 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (5) CometFilter Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] @@ -218,12 +232,15 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) ColumnarToRow [codegen id : 1] +(11) CometColumnarToRow [codegen id : 1] Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(12) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct (13) CometFilter Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] @@ -241,9 +258,12 @@ Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(17) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(17) CometNativeScan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct (18) CometFilter Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] @@ -270,667 +290,724 @@ Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number# Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(24) ColumnarToRow [codegen id : 2] -Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(25) HashAggregate [codegen id : 2] +(24) CometHashAggregate Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] Keys [1]: [cs_item_sk#16] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -(26) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(25) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(27) HashAggregate [codegen id : 3] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] + +(27) HashAggregate [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] Keys [1]: [cs_item_sk#16] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#29, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30] +Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#29,17,2) AS sale#31, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30 AS refund#32] -(28) Filter [codegen id : 3] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) +(28) Filter [codegen id : 2] +Input [3]: [cs_item_sk#16, sale#31, refund#32] +Condition : ((isnotnull(sale#31) AND isnotnull(refund#32)) AND (cast(sale#31 as decimal(21,2)) > (2 * refund#32))) -(29) Project [codegen id : 3] +(29) Project [codegen id : 2] Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] +Input [3]: [cs_item_sk#16, sale#31, refund#32] -(30) Sort [codegen id : 3] +(30) Sort [codegen id : 2] Input [1]: [cs_item_sk#16] Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 -(31) SortMergeJoin [codegen id : 19] +(31) SortMergeJoin [codegen id : 18] Left keys [1]: [ss_item_sk#1] Right keys [1]: [cs_item_sk#16] Join type: Inner Join condition: None -(32) Project [codegen id : 19] +(32) Project [codegen id : 18] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#36, d_year#37] -Arguments: [d_date_sk#36, d_year#37] +(33) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#33, d_year#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (34) CometFilter -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) +Input [2]: [d_date_sk#33, d_year#34] +Condition : ((isnotnull(d_year#34) AND (d_year#34 = 1999)) AND isnotnull(d_date_sk#33)) -(35) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#36, d_year#37] +(35) CometColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#33, d_year#34] (36) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] +Input [2]: [d_date_sk#33, d_year#34] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(37) BroadcastHashJoin [codegen id : 19] +(37) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] +Right keys [1]: [d_date_sk#33] Join type: Inner Join condition: None -(38) Project [codegen id : 19] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] +(38) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#33, d_year#34] -(39) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: [s_store_sk#38, s_store_name#39, s_zip#40] +(39) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] +ReadSchema: struct (40) CometFilter -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Condition : ((isnotnull(s_store_sk#35) AND isnotnull(s_store_name#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)))) + +(41) CometProject +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Arguments: [s_store_sk#35, s_store_name#36, s_zip#38], [s_store_sk#35, s_store_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)) AS s_zip#38] -(41) ColumnarToRow [codegen id : 5] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(42) CometColumnarToRow [codegen id : 4] +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] -(42) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(43) BroadcastExchange +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(43) BroadcastHashJoin [codegen id : 19] +(44) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] +Right keys [1]: [s_store_sk#35] Join type: Inner Join condition: None -(44) Project [codegen id : 19] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] +(45) Project [codegen id : 18] +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_sk#35, s_store_name#36, s_zip#38] -(45) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(46) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct -(46) CometFilter -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) +(47) CometFilter +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Condition : (((((isnotnull(c_customer_sk#39) AND isnotnull(c_first_sales_date_sk#44)) AND isnotnull(c_first_shipto_date_sk#43)) AND isnotnull(c_current_cdemo_sk#40)) AND isnotnull(c_current_hdemo_sk#41)) AND isnotnull(c_current_addr_sk#42)) -(47) ColumnarToRow [codegen id : 6] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(48) CometColumnarToRow [codegen id : 5] +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] -(48) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(49) BroadcastExchange +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] -(49) BroadcastHashJoin [codegen id : 19] +(50) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] +Right keys [1]: [c_customer_sk#39] Join type: Inner Join condition: None -(50) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(51) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] -(51) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#47, d_year#48] -Arguments: [d_date_sk#47, d_year#48] +(52) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#45, d_year#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct -(52) CometFilter -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) +(53) CometFilter +Input [2]: [d_date_sk#45, d_year#46] +Condition : isnotnull(d_date_sk#45) -(53) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#47, d_year#48] +(54) CometColumnarToRow [codegen id : 6] +Input [2]: [d_date_sk#45, d_year#46] -(54) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] +(55) BroadcastExchange +Input [2]: [d_date_sk#45, d_year#46] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -(55) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] +(56) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_sales_date_sk#44] +Right keys [1]: [d_date_sk#45] Join type: Inner Join condition: None -(56) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] +(57) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44, d_date_sk#45, d_year#46] -(57) ReusedExchange [Reuses operator id: 54] -Output [2]: [d_date_sk#49, d_year#50] +(58) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#47, d_year#48] -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] +(59) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_shipto_date_sk#43] +Right keys [1]: [d_date_sk#47] Join type: Inner Join condition: None -(59) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] +(60) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46, d_date_sk#47, d_year#48] -(60) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: [cd_demo_sk#51, cd_marital_status#52] +(61) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#49, cd_marital_status#50] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(61) CometFilter -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) +(62) CometFilter +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Condition : (isnotnull(cd_demo_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)))) -(62) ColumnarToRow [codegen id : 9] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] +(63) CometProject +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Arguments: [cd_demo_sk#49, cd_marital_status#51], [cd_demo_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)) AS cd_marital_status#51] -(63) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(64) CometColumnarToRow [codegen id : 8] +Input [2]: [cd_demo_sk#49, cd_marital_status#51] -(64) BroadcastHashJoin [codegen id : 19] +(65) BroadcastExchange +Input [2]: [cd_demo_sk#49, cd_marital_status#51] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(66) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] +Right keys [1]: [cd_demo_sk#49] Join type: Inner Join condition: None -(65) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] +(67) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_demo_sk#49, cd_marital_status#51] -(66) ReusedExchange [Reuses operator id: 63] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] +(68) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#52, cd_marital_status#53] -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] +(69) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_cdemo_sk#40] +Right keys [1]: [cd_demo_sk#52] Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) +Join condition: NOT (cd_marital_status#51 = cd_marital_status#53) -(68) Project [codegen id : 19] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] +(70) Project [codegen id : 18] +Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51, cd_demo_sk#52, cd_marital_status#53] -(69) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#55] -Arguments: [p_promo_sk#55] +(71) CometNativeScan parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct -(70) CometFilter -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) +(72) CometFilter +Input [1]: [p_promo_sk#54] +Condition : isnotnull(p_promo_sk#54) -(71) ColumnarToRow [codegen id : 11] -Input [1]: [p_promo_sk#55] +(73) CometColumnarToRow [codegen id : 10] +Input [1]: [p_promo_sk#54] -(72) BroadcastExchange -Input [1]: [p_promo_sk#55] +(74) BroadcastExchange +Input [1]: [p_promo_sk#54] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -(73) BroadcastHashJoin [codegen id : 19] +(75) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] +Right keys [1]: [p_promo_sk#54] Join type: Inner Join condition: None -(74) Project [codegen id : 19] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] +(76) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, p_promo_sk#54] -(75) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [hd_demo_sk#56, hd_income_band_sk#57] +(77) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct -(76) CometFilter -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) +(78) CometFilter +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Condition : (isnotnull(hd_demo_sk#55) AND isnotnull(hd_income_band_sk#56)) -(77) ColumnarToRow [codegen id : 12] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(79) CometColumnarToRow [codegen id : 11] +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(78) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(80) BroadcastExchange +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] -(79) BroadcastHashJoin [codegen id : 19] +(81) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] +Right keys [1]: [hd_demo_sk#55] Join type: Inner Join condition: None -(80) Project [codegen id : 19] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] +(82) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_demo_sk#55, hd_income_band_sk#56] -(81) ReusedExchange [Reuses operator id: 78] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] +(83) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] -(82) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] +(84) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_hdemo_sk#41] +Right keys [1]: [hd_demo_sk#57] Join type: Inner Join condition: None -(83) Project [codegen id : 19] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] +(85) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_demo_sk#57, hd_income_band_sk#58] + +(86) CometNativeScan parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(84) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(87) CometFilter +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Condition : isnotnull(ca_address_sk#59) -(85) CometFilter -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) +(88) CometProject +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Arguments: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65], [ca_address_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#60, 10)) AS ca_street_number#64, ca_street_name#61, ca_city#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#63, 10)) AS ca_zip#65] -(86) ColumnarToRow [codegen id : 14] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(89) CometColumnarToRow [codegen id : 13] +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] -(87) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(90) BroadcastExchange +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -(88) BroadcastHashJoin [codegen id : 19] +(91) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] +Right keys [1]: [ca_address_sk#59] Join type: Inner Join condition: None -(89) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(92) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] -(90) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(93) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(91) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] +(94) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_addr_sk#42] +Right keys [1]: [ca_address_sk#66] Join type: Inner Join condition: None -(92) Project [codegen id : 19] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(95) Project [codegen id : 18] +Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] + +(96) CometNativeScan parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#71] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(97) CometFilter +Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) + +(98) CometColumnarToRow [codegen id : 15] +Input [1]: [ib_income_band_sk#71] -(93) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#70] +(99) BroadcastExchange +Input [1]: [ib_income_band_sk#71] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] -(94) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] +(100) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#56] +Right keys [1]: [ib_income_band_sk#71] Join type: Inner Join condition: None -(95) Project [codegen id : 19] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] +(101) Project [codegen id : 18] +Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] -(96) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#71] +(102) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#72] -(97) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] +(103) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#58] +Right keys [1]: [ib_income_band_sk#72] Join type: Inner Join condition: None -(98) Project [codegen id : 19] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] +(104) Project [codegen id : 18] +Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] -(99) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(105) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +ReadSchema: struct -(100) CometFilter -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +(106) CometFilter +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#75, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(101) CometProject -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_product_name#75], [i_item_sk#72, i_product_name#75] +(107) CometProject +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#77], [i_item_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#76, 50)) AS i_product_name#77] -(102) ColumnarToRow [codegen id : 18] -Input [2]: [i_item_sk#72, i_product_name#75] +(108) CometColumnarToRow [codegen id : 17] +Input [2]: [i_item_sk#73, i_product_name#77] -(103) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +(109) BroadcastExchange +Input [2]: [i_item_sk#73, i_product_name#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] -(104) BroadcastHashJoin [codegen id : 19] +(110) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] +Right keys [1]: [i_item_sk#73] Join type: Inner Join condition: None -(105) Project [codegen id : 19] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] +(111) Project [codegen id : 18] +Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] -(106) HashAggregate [codegen id : 19] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(112) HashAggregate [codegen id : 18] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] +Aggregate Attributes [4]: [count#78, sum#79, sum#80, sum#81] +Results [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] -(107) HashAggregate [codegen id : 19] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(113) HashAggregate [codegen id : 18] +Input [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#9))#87, sum(UnscaledValue(ss_list_price#10))#88, sum(UnscaledValue(ss_coupon_amt#11))#89] +Results [17]: [i_product_name#77 AS product_name#90, i_item_sk#73 AS item_sk#91, s_store_name#36 AS store_name#92, s_zip#38 AS store_zip#93, ca_street_number#64 AS b_street_number#94, ca_street_name#61 AS b_streen_name#95, ca_city#62 AS b_city#96, ca_zip#65 AS b_zip#97, ca_street_number#67 AS c_street_number#98, ca_street_name#68 AS c_street_name#99, ca_city#69 AS c_city#100, ca_zip#70 AS c_zip#101, d_year#34 AS syear#102, count(1)#86 AS cnt#103, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#87,17,2) AS s1#104, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#88,17,2) AS s2#105, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#89,17,2) AS s3#106] + +(114) CometColumnarExchange +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: hashpartitioning(item_sk#91, store_name#92, store_zip#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] -(108) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=14] +(115) CometSort +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106], [item_sk#91 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, store_zip#93 ASC NULLS FIRST] -(109) Sort [codegen id : 20] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 +(116) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(110) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] +(117) CometSort +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Arguments: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#107 ASC NULLS FIRST] -(111) CometSort -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115], [ss_item_sk#105 ASC NULLS FIRST] +(118) CometColumnarToRow [codegen id : 19] +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(112) ColumnarToRow [codegen id : 21] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] +(119) ReusedExchange [Reuses operator id: 25] +Output [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] -(113) ReusedExchange [Reuses operator id: 26] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] +(120) CometColumnarToRow [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] -(114) HashAggregate [codegen id : 23] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#124, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#125] +(121) HashAggregate [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] +Keys [1]: [cs_item_sk#118] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#122)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#122))#29, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30] +Results [3]: [cs_item_sk#118, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#122))#29,17,2) AS sale#126, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30 AS refund#127] -(115) Filter [codegen id : 23] -Input [3]: [cs_item_sk#116, sale#124, refund#125] -Condition : ((isnotnull(sale#124) AND isnotnull(refund#125)) AND (cast(sale#124 as decimal(21,2)) > (2 * refund#125))) +(122) Filter [codegen id : 20] +Input [3]: [cs_item_sk#118, sale#126, refund#127] +Condition : ((isnotnull(sale#126) AND isnotnull(refund#127)) AND (cast(sale#126 as decimal(21,2)) > (2 * refund#127))) -(116) Project [codegen id : 23] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#124, refund#125] +(123) Project [codegen id : 20] +Output [1]: [cs_item_sk#118] +Input [3]: [cs_item_sk#118, sale#126, refund#127] -(117) Sort [codegen id : 23] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 +(124) Sort [codegen id : 20] +Input [1]: [cs_item_sk#118] +Arguments: [cs_item_sk#118 ASC NULLS FIRST], false, 0 -(118) SortMergeJoin [codegen id : 39] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] +(125) SortMergeJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [cs_item_sk#118] Join type: Inner Join condition: None -(119) Project [codegen id : 39] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(120) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#126, d_year#127] -Arguments: [d_date_sk#126, d_year#127] - -(121) CometFilter -Input [2]: [d_date_sk#126, d_year#127] -Condition : ((isnotnull(d_year#127) AND (d_year#127 = 2000)) AND isnotnull(d_date_sk#126)) +(126) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Input [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#118] -(122) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#126, d_year#127] +(127) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#128, d_year#129] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct -(123) BroadcastExchange -Input [2]: [d_date_sk#126, d_year#127] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] +(128) CometFilter +Input [2]: [d_date_sk#128, d_year#129] +Condition : ((isnotnull(d_year#129) AND (d_year#129 = 2000)) AND isnotnull(d_date_sk#128)) -(124) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#126] -Join type: Inner -Join condition: None +(129) CometColumnarToRow [codegen id : 21] +Input [2]: [d_date_sk#128, d_year#129] -(125) Project [codegen id : 39] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#126, d_year#127] +(130) BroadcastExchange +Input [2]: [d_date_sk#128, d_year#129] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] -(126) ReusedExchange [Reuses operator id: 42] -Output [3]: [s_store_sk#128, s_store_name#129, s_zip#130] - -(127) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#128] +(131) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_sold_date_sk#117] +Right keys [1]: [d_date_sk#128] Join type: Inner Join condition: None -(128) Project [codegen id : 39] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_sk#128, s_store_name#129, s_zip#130] +(132) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129] +Input [13]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#128, d_year#129] -(129) ReusedExchange [Reuses operator id: 48] -Output [6]: [c_customer_sk#131, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136] +(133) ReusedExchange [Reuses operator id: 43] +Output [3]: [s_store_sk#130, s_store_name#131, s_zip#132] -(130) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#131] +(134) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_store_sk#112] +Right keys [1]: [s_store_sk#130] Join type: Inner Join condition: None -(131) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_customer_sk#131, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136] +(135) Project [codegen id : 36] +Output [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132] +Input [14]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_sk#130, s_store_name#131, s_zip#132] -(132) ReusedExchange [Reuses operator id: 54] -Output [2]: [d_date_sk#137, d_year#138] +(136) ReusedExchange [Reuses operator id: 49] +Output [6]: [c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] -(133) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_first_sales_date_sk#136] -Right keys [1]: [d_date_sk#137] +(137) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_customer_sk#108] +Right keys [1]: [c_customer_sk#133] Join type: Inner Join condition: None -(134) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136, d_date_sk#137, d_year#138] +(138) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] +Input [18]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] -(135) ReusedExchange [Reuses operator id: 54] +(139) ReusedExchange [Reuses operator id: 55] Output [2]: [d_date_sk#139, d_year#140] -(136) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_first_shipto_date_sk#135] +(140) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_sales_date_sk#138] Right keys [1]: [d_date_sk#139] Join type: Inner Join condition: None -(137) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, d_year#138, d_date_sk#139, d_year#140] +(141) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138, d_date_sk#139, d_year#140] -(138) ReusedExchange [Reuses operator id: 63] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] +(142) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#141, d_year#142] -(139) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#141] +(143) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_shipto_date_sk#137] +Right keys [1]: [d_date_sk#141] Join type: Inner Join condition: None -(140) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, cd_marital_status#142] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, cd_demo_sk#141, cd_marital_status#142] +(144) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140, d_date_sk#141, d_year#142] -(141) ReusedExchange [Reuses operator id: 63] +(145) ReusedExchange [Reuses operator id: 65] Output [2]: [cd_demo_sk#143, cd_marital_status#144] -(142) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_cdemo_sk#132] +(146) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_cdemo_sk#109] Right keys [1]: [cd_demo_sk#143] Join type: Inner -Join condition: NOT (cd_marital_status#142 = cd_marital_status#144) +Join condition: None -(143) Project [codegen id : 39] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, cd_marital_status#142, cd_demo_sk#143, cd_marital_status#144] +(147) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_demo_sk#143, cd_marital_status#144] -(144) ReusedExchange [Reuses operator id: 72] -Output [1]: [p_promo_sk#145] +(148) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] -(145) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#145] +(149) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_cdemo_sk#134] +Right keys [1]: [cd_demo_sk#145] Join type: Inner -Join condition: None +Join condition: NOT (cd_marital_status#144 = cd_marital_status#146) -(146) Project [codegen id : 39] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, p_promo_sk#145] +(150) Project [codegen id : 36] +Output [14]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144, cd_demo_sk#145, cd_marital_status#146] -(147) ReusedExchange [Reuses operator id: 78] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] +(151) ReusedExchange [Reuses operator id: 74] +Output [1]: [p_promo_sk#147] -(148) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#146] +(152) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_promo_sk#113] +Right keys [1]: [p_promo_sk#147] Join type: Inner Join condition: None -(149) Project [codegen id : 39] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, hd_demo_sk#146, hd_income_band_sk#147] +(153) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, p_promo_sk#147] -(150) ReusedExchange [Reuses operator id: 78] +(154) ReusedExchange [Reuses operator id: 80] Output [2]: [hd_demo_sk#148, hd_income_band_sk#149] -(151) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_hdemo_sk#133] +(155) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_hdemo_sk#110] Right keys [1]: [hd_demo_sk#148] Join type: Inner Join condition: None -(152) Project [codegen id : 39] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_demo_sk#148, hd_income_band_sk#149] +(156) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_demo_sk#148, hd_income_band_sk#149] -(153) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#150, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154] +(157) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] -(154) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#150] +(158) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_hdemo_sk#135] +Right keys [1]: [hd_demo_sk#150] Join type: Inner Join condition: None -(155) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_address_sk#150, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154] +(159) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151] +Input [15]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_demo_sk#150, hd_income_band_sk#151] -(156) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#155, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] +(160) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] -(157) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_addr_sk#134] -Right keys [1]: [ca_address_sk#155] +(161) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_addr_sk#111] +Right keys [1]: [ca_address_sk#152] Join type: Inner Join condition: None -(158) Project [codegen id : 39] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_address_sk#155, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] +(162) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] +Input [18]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] -(159) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#160] +(163) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] -(160) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#160] +(164) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_addr_sk#136] +Right keys [1]: [ca_address_sk#157] Join type: Inner Join condition: None -(161) Project [codegen id : 39] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, ib_income_band_sk#160] +(165) Project [codegen id : 36] +Output [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [21]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] -(162) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#161] +(166) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#162] -(163) BroadcastHashJoin [codegen id : 39] +(167) BroadcastHashJoin [codegen id : 36] Left keys [1]: [hd_income_band_sk#149] -Right keys [1]: [ib_income_band_sk#161] +Right keys [1]: [ib_income_band_sk#162] Join type: Inner Join condition: None -(164) Project [codegen id : 39] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, ib_income_band_sk#161] +(168) Project [codegen id : 36] +Output [18]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [20]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#162] -(165) ReusedExchange [Reuses operator id: 103] -Output [2]: [i_item_sk#162, i_product_name#163] +(169) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#163] -(166) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#162] +(170) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#163] Join type: Inner Join condition: None -(167) Project [codegen id : 39] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, d_year#138, d_year#140, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, i_item_sk#162, i_product_name#163] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, i_item_sk#162, i_product_name#163] - -(168) HashAggregate [codegen id : 39] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, d_year#138, d_year#140, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, i_item_sk#162, i_product_name#163] -Keys [15]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#164, sum#165, sum#166] -Results [19]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140, count#80, sum#167, sum#168, sum#169] - -(169) HashAggregate [codegen id : 39] -Input [19]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140, count#80, sum#167, sum#168, sum#169] -Keys [15]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#162 AS item_sk#170, s_store_name#129 AS store_name#171, s_zip#130 AS store_zip#172, d_year#127 AS syear#173, count(1)#84 AS cnt#174, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#175, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#176, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#177] - -(170) Exchange -Input [8]: [item_sk#170, store_name#171, store_zip#172, syear#173, cnt#174, s1#175, s2#176, s3#177] -Arguments: hashpartitioning(item_sk#170, store_name#171, store_zip#172, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(171) Project [codegen id : 36] +Output [17]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#163] -(171) Sort [codegen id : 40] -Input [8]: [item_sk#170, store_name#171, store_zip#172, syear#173, cnt#174, s1#175, s2#176, s3#177] -Arguments: [item_sk#170 ASC NULLS FIRST, store_name#171 ASC NULLS FIRST, store_zip#172 ASC NULLS FIRST], false, 0 +(172) ReusedExchange [Reuses operator id: 109] +Output [2]: [i_item_sk#164, i_product_name#165] -(172) SortMergeJoin [codegen id : 41] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#170, store_name#171, store_zip#172] +(173) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [i_item_sk#164] Join type: Inner -Join condition: (cnt#174 <= cnt#101) - -(173) Project [codegen id : 41] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#170, store_name#171, store_zip#172, syear#173, cnt#174, s1#175, s2#176, s3#177] - -(174) CometColumnarExchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#174 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] - -(175) CometSort -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] -Arguments: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174], [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#174 ASC NULLS FIRST] +Join condition: None -(176) ColumnarToRow [codegen id : 42] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] +(174) Project [codegen id : 36] +Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] + +(175) HashAggregate [codegen id : 36] +Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count#78, sum#166, sum#167, sum#168] +Results [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] + +(176) HashAggregate [codegen id : 36] +Input [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#114))#87, sum(UnscaledValue(ss_list_price#115))#88, sum(UnscaledValue(ss_coupon_amt#116))#89] +Results [8]: [i_item_sk#164 AS item_sk#172, s_store_name#131 AS store_name#173, s_zip#132 AS store_zip#174, d_year#129 AS syear#175, count(1)#86 AS cnt#176, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#87,17,2) AS s1#177, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#88,17,2) AS s2#178, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#89,17,2) AS s3#179] + +(177) CometColumnarExchange +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: hashpartitioning(item_sk#172, store_name#173, store_zip#174, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] + +(178) CometSort +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179], [item_sk#172 ASC NULLS FIRST, store_name#173 ASC NULLS FIRST, store_zip#174 ASC NULLS FIRST] + +(179) CometSortMergeJoin +Left output [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Right output [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#91, store_name#92, store_zip#93], [item_sk#172, store_name#173, store_zip#174], Inner, (cnt#176 <= cnt#103) + +(180) CometProject +Input [25]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + +(181) CometExchange +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: rangepartitioning(product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] + +(182) CometSort +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST] + +(183) CometColumnarToRow [codegen id : 37] +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/extended.txt new file mode 100644 index 0000000000..0ca4716e97 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/extended.txt @@ -0,0 +1,271 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Project + : : : : +- BroadcastHashJoin + : : : : :- Project + : : : : : +- BroadcastHashJoin + : : : : : :- Project + : : : : : : +- BroadcastHashJoin + : : : : : : :- Project + : : : : : : : +- BroadcastHashJoin + : : : : : : : :- Project + : : : : : : : : +- BroadcastHashJoin + : : : : : : : : :- Project + : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : :- Project + : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : :- Project + : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : :- Project + : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : : : +- SortMergeJoin + : : : : : : : : : : : : : : : : :- CometColumnarToRow + : : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- Sort + : : : : : : : : : : : : : : : : +- Project + : : : : : : : : : : : : : : : : +- Filter + : : : : : : : : : : : : : : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_item_sk#1, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#2))#3,17,2) AS sale#4, sum(((cr_refunded_cash#5 + cr_reversed_charge#6) + cr_store_credit#7))#8 AS refund#9)] + : : : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : +- CometProject + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- BroadcastExchange + : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- BroadcastExchange + : : : : : : : : +- CometColumnarToRow + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : : +- BroadcastExchange + : : : : : : : +- CometColumnarToRow + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : : +- BroadcastExchange + : : : : : : +- CometColumnarToRow + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- BroadcastExchange + : : : : : +- CometColumnarToRow + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- BroadcastExchange + : : : : +- CometColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.income_band + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Project + : : : : +- BroadcastHashJoin + : : : : :- Project + : : : : : +- BroadcastHashJoin + : : : : : :- Project + : : : : : : +- BroadcastHashJoin + : : : : : : :- Project + : : : : : : : +- BroadcastHashJoin + : : : : : : : :- Project + : : : : : : : : +- BroadcastHashJoin + : : : : : : : : :- Project + : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : :- Project + : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : :- Project + : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : :- Project + : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : : +- SortMergeJoin + : : : : : : : : : : : : : : : :- CometColumnarToRow + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : +- Sort + : : : : : : : : : : : : : : : +- Project + : : : : : : : : : : : : : : : +- Filter + : : : : : : : : : : : : : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_item_sk#10, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#11))#3,17,2) AS sale#12, sum(((cr_refunded_cash#13 + cr_reversed_charge#14) + cr_store_credit#15))#8 AS refund#16)] + : : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : +- BroadcastExchange + : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- BroadcastExchange + : : : : : : : : +- CometColumnarToRow + : : : : : : : : +- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : +- BroadcastExchange + : : : : : : : +- CometColumnarToRow + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : +- BroadcastExchange + : : : : : : +- CometColumnarToRow + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- BroadcastExchange + : : : : : +- CometColumnarToRow + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- BroadcastExchange + : : : : +- CometColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 128 out of 234 eligible operators (54%). Final plan contains 35 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/simplified.txt index dae7ae193a..80c873ed48 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/simplified.txt @@ -1,251 +1,251 @@ -WholeStageCodegen (42) - ColumnarToRow +WholeStageCodegen (37) + CometColumnarToRow InputAdapter CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt] #1 - WholeStageCodegen (41) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (20) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (19) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (3) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (2) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #6 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #7 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometExchange [product_name,store_name,cnt] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #2 + WholeStageCodegen (18) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] InputAdapter - BroadcastExchange #9 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + InputAdapter + WholeStageCodegen (2) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter - BroadcastExchange #10 - WholeStageCodegen (6) - ColumnarToRow + BroadcastExchange #8 + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (7) - ColumnarToRow + BroadcastExchange #9 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] InputAdapter - ReusedExchange [d_date_sk,d_year] #11 + BroadcastExchange #10 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - ColumnarToRow + BroadcastExchange #11 + WholeStageCodegen (6) + CometColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [d_date_sk,d_year] #11 InputAdapter - BroadcastExchange #13 - WholeStageCodegen (11) - ColumnarToRow + BroadcastExchange #12 + WholeStageCodegen (8) + CometColumnarToRow InputAdapter - CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter - BroadcastExchange #14 - WholeStageCodegen (12) - ColumnarToRow - InputAdapter - CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] + ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + BroadcastExchange #13 + WholeStageCodegen (10) + CometColumnarToRow + InputAdapter + CometFilter [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] InputAdapter - BroadcastExchange #15 - WholeStageCodegen (14) - ColumnarToRow + BroadcastExchange #14 + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter - ReusedExchange [ib_income_band_sk] #13 + BroadcastExchange #15 + WholeStageCodegen (13) + CometColumnarToRow + InputAdapter + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] InputAdapter - ReusedExchange [ib_income_band_sk] #13 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter BroadcastExchange #16 - WholeStageCodegen (18) - ColumnarToRow + WholeStageCodegen (15) + CometColumnarToRow InputAdapter - CometProject [i_item_sk,i_product_name] - CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (40) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #17 - WholeStageCodegen (39) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (21) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (23) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #18 - WholeStageCodegen (24) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometFilter [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + BroadcastExchange #17 + WholeStageCodegen (17) + CometColumnarToRow + InputAdapter + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #18 + WholeStageCodegen (36) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + InputAdapter + WholeStageCodegen (20) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + BroadcastExchange #19 + WholeStageCodegen (21) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 InputAdapter - ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [d_date_sk,d_year] #11 InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [d_date_sk,d_year] #11 InputAdapter - ReusedExchange [p_promo_sk] #13 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [p_promo_sk] #13 InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter - ReusedExchange [ib_income_band_sk] #13 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter - ReusedExchange [ib_income_band_sk] #13 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter - ReusedExchange [i_item_sk,i_product_name] #16 + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/explain.txt index 5b68aeb7f2..6d09bc0575 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/explain.txt @@ -1,70 +1,82 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) +TakeOrderedAndProject (44) ++- * Project (43) + +- * BroadcastHashJoin Inner BuildRight (42) + :- * Project (26) + : +- * BroadcastHashJoin Inner BuildRight (25) : :- * Project (19) : : +- * BroadcastHashJoin Inner BuildRight (18) - : : :- * ColumnarToRow (3) + : : :- * CometColumnarToRow (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (1) + : : : +- CometNativeScan parquet spark_catalog.default.store (1) : : +- BroadcastExchange (17) : : +- * Filter (16) : : +- * HashAggregate (15) - : : +- Exchange (14) - : : +- * HashAggregate (13) - : : +- * ColumnarToRow (12) + : : +- * CometColumnarToRow (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) : : +- CometProject (11) : : +- CometBroadcastHashJoin (10) : : :- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) : : +- CometBroadcastExchange (9) : : +- CometProject (8) : : +- CometFilter (7) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (6) - : +- BroadcastExchange (23) - : +- * ColumnarToRow (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - +- BroadcastExchange (39) - +- * Filter (38) - +- * HashAggregate (37) - +- Exchange (36) - +- * HashAggregate (35) - +- * HashAggregate (34) - +- Exchange (33) - +- * HashAggregate (32) - +- * ColumnarToRow (31) - +- CometProject (30) - +- CometBroadcastHashJoin (29) - :- CometFilter (27) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) - +- ReusedExchange (28) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (6) + : +- BroadcastExchange (24) + : +- * CometColumnarToRow (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- BroadcastExchange (41) + +- * Filter (40) + +- * HashAggregate (39) + +- * CometColumnarToRow (38) + +- CometColumnarExchange (37) + +- * HashAggregate (36) + +- * HashAggregate (35) + +- * CometColumnarToRow (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (28) + : +- CometNativeScan parquet spark_catalog.default.store_sales (27) + +- ReusedExchange (29) + + +(1) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#1, s_store_name#2] -Arguments: [s_store_sk#1, s_store_name#2] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(3) ColumnarToRow [codegen id : 7] +(3) CometColumnarToRow [codegen id : 5] Input [2]: [s_store_sk#1, s_store_name#2] -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(6) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(6) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_month_seq#8] -Arguments: [d_date_sk#7, d_month_seq#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] +ReadSchema: struct (7) CometFilter Input [2]: [d_date_sk#7, d_month_seq#8] @@ -87,148 +99,158 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] -(12) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] - -(13) HashAggregate [codegen id : 1] +(12) CometHashAggregate Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -(14) Exchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(13) CometExchange +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] +Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(15) HashAggregate [codegen id : 2] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +(14) CometColumnarToRow [codegen id : 1] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] + +(15) HashAggregate [codegen id : 1] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#11] -Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#11,17,2) AS revenue#12] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#10] +Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#10,17,2) AS revenue#11] -(16) Filter [codegen id : 2] -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Condition : isnotnull(revenue#12) +(16) Filter [codegen id : 1] +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Condition : isnotnull(revenue#11) (17) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(18) BroadcastHashJoin [codegen id : 7] +(18) BroadcastHashJoin [codegen id : 5] Left keys [1]: [s_store_sk#1] Right keys [1]: [ss_store_sk#4] Join type: Inner Join condition: None -(19) Project [codegen id : 7] -Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] +(19) Project [codegen id : 5] +Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] +Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(20) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (21) CometFilter -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Condition : isnotnull(i_item_sk#13) +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Condition : isnotnull(i_item_sk#12) + +(22) CometProject +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Arguments: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#16, 50)) AS i_brand#17] -(22) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(23) CometColumnarToRow [codegen id : 2] +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -(23) BroadcastExchange -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(24) BroadcastExchange +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(24) BroadcastHashJoin [codegen id : 7] +(25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#3] -Right keys [1]: [i_item_sk#13] +Right keys [1]: [i_item_sk#12] Join type: Inner Join condition: None -(25) Project [codegen id : 7] -Output [7]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12, i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(26) Project [codegen id : 5] +Output [7]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11, i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(27) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#21)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct -(27) CometFilter +(28) CometFilter Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] Condition : isnotnull(ss_store_sk#19) -(28) ReusedExchange [Reuses operator id: 9] +(29) ReusedExchange [Reuses operator id: 9] Output [1]: [d_date_sk#22] -(29) CometBroadcastHashJoin +(30) CometBroadcastHashJoin Left output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] Right output [1]: [d_date_sk#22] Arguments: [ss_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight -(30) CometProject +(31) CometProject Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#22] Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20], [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -(31) ColumnarToRow [codegen id : 4] -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] - -(32) HashAggregate [codegen id : 4] +(32) CometHashAggregate Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] Keys [2]: [ss_store_sk#19, ss_item_sk#18] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum#23] -Results [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -(33) Exchange -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(33) CometExchange +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] +Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(34) HashAggregate [codegen id : 5] -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] +(34) CometColumnarToRow [codegen id : 3] +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] + +(35) HashAggregate [codegen id : 3] +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] Keys [2]: [ss_store_sk#19, ss_item_sk#18] Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#25] -Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#25,17,2) AS revenue#26] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#24] +Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#24,17,2) AS revenue#25] -(35) HashAggregate [codegen id : 5] -Input [2]: [ss_store_sk#19, revenue#26] +(36) HashAggregate [codegen id : 3] +Input [2]: [ss_store_sk#19, revenue#25] Keys [1]: [ss_store_sk#19] -Functions [1]: [partial_avg(revenue#26)] -Aggregate Attributes [2]: [sum#27, count#28] -Results [3]: [ss_store_sk#19, sum#29, count#30] +Functions [1]: [partial_avg(revenue#25)] +Aggregate Attributes [2]: [sum#26, count#27] +Results [3]: [ss_store_sk#19, sum#28, count#29] + +(37) CometColumnarExchange +Input [3]: [ss_store_sk#19, sum#28, count#29] +Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(36) Exchange -Input [3]: [ss_store_sk#19, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(38) CometColumnarToRow [codegen id : 4] +Input [3]: [ss_store_sk#19, sum#28, count#29] -(37) HashAggregate [codegen id : 6] -Input [3]: [ss_store_sk#19, sum#29, count#30] +(39) HashAggregate [codegen id : 4] +Input [3]: [ss_store_sk#19, sum#28, count#29] Keys [1]: [ss_store_sk#19] -Functions [1]: [avg(revenue#26)] -Aggregate Attributes [1]: [avg(revenue#26)#31] -Results [2]: [ss_store_sk#19, avg(revenue#26)#31 AS ave#32] +Functions [1]: [avg(revenue#25)] +Aggregate Attributes [1]: [avg(revenue#25)#30] +Results [2]: [ss_store_sk#19, avg(revenue#25)#30 AS ave#31] -(38) Filter [codegen id : 6] -Input [2]: [ss_store_sk#19, ave#32] -Condition : isnotnull(ave#32) +(40) Filter [codegen id : 4] +Input [2]: [ss_store_sk#19, ave#31] +Condition : isnotnull(ave#31) -(39) BroadcastExchange -Input [2]: [ss_store_sk#19, ave#32] +(41) BroadcastExchange +Input [2]: [ss_store_sk#19, ave#31] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(40) BroadcastHashJoin [codegen id : 7] +(42) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#4] Right keys [1]: [ss_store_sk#19] Join type: Inner -Join condition: (cast(revenue#12 as decimal(23,7)) <= (0.1 * ave#32)) +Join condition: (cast(revenue#11 as decimal(23,7)) <= (0.1 * ave#31)) -(41) Project [codegen id : 7] -Output [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17, ss_store_sk#19, ave#32] +(43) Project [codegen id : 5] +Output [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17, ss_store_sk#19, ave#31] -(42) TakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#14 ASC NULLS FIRST], [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(44) TakeOrderedAndProject +Input [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#13 ASC NULLS FIRST], [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/extended.txt new file mode 100644 index 0000000000..2db2532726 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/extended.txt @@ -0,0 +1,49 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- CometColumnarToRow + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- BroadcastExchange + : : +- Filter + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_store_sk#1, ss_item_sk#2, MakeDecimal(sum(UnscaledValue(ss_sales_price#3))#4,17,2) AS revenue#5)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- Filter + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_store_sk#6, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#8,17,2) AS revenue#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 26 out of 42 eligible operators (61%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/simplified.txt index bf94064740..2c1cd9fb8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/simplified.txt @@ -1,59 +1,57 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - WholeStageCodegen (7) + WholeStageCodegen (5) Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave] Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] BroadcastHashJoin [ss_item_sk,i_item_sk] Project [s_store_name,ss_store_sk,ss_item_sk,revenue] BroadcastHashJoin [s_store_sk,ss_store_sk] - ColumnarToRow + CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #1 - WholeStageCodegen (2) + WholeStageCodegen (1) Filter [revenue] HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #2 - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometColumnarToRow + InputAdapter + CometExchange [ss_store_sk,ss_item_sk] #2 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #4 - WholeStageCodegen (3) - ColumnarToRow + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] InputAdapter BroadcastExchange #5 - WholeStageCodegen (6) + WholeStageCodegen (4) Filter [ave] HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] - InputAdapter - Exchange [ss_store_sk] #6 - WholeStageCodegen (5) - HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #7 - WholeStageCodegen (4) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #3 + CometColumnarToRow + InputAdapter + CometColumnarExchange [ss_store_sk] #6 + WholeStageCodegen (3) + HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] + HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_store_sk,ss_item_sk] #7 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/explain.txt index 127adabf18..0ef9979fa1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/explain.txt @@ -1,193 +1,287 @@ == Physical Plan == -TakeOrderedAndProject (35) -+- * HashAggregate (34) - +- Exchange (33) - +- * HashAggregate (32) - +- Union (31) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (13) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`ship_mode` (19) - +- * HashAggregate (30) - +- ReusedExchange (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* CometColumnarToRow (51) ++- CometTakeOrderedAndProject (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometUnion (46) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.ship_mode (20) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] +PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Condition : isnotnull(w_warehouse_sk#8) -(5) CometBroadcastExchange +(5) CometProject Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#13, 2)) AS w_state#15, w_country#14] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] + +(7) CometBroadcastHashJoin Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight -(7) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(9) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(11) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Right output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(13) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18, t_time#19] - -(14) CometFilter -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) - -(15) CometProject -Input [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18], [t_time_sk#18] - -(16) CometBroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: [t_time_sk#18] - -(17) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [t_time_sk#18] -Arguments: [ws_sold_time_sk#1], [t_time_sk#18], Inner, BuildRight - -(18) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(19) CometNativeScan: `spark_catalog`.`default`.`ship_mode` -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20, sm_carrier#21] - -(20) CometFilter -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) - -(21) CometProject -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20], [sm_ship_mode_sk#20] - -(22) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: [sm_ship_mode_sk#20] - -(23) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [sm_ship_mode_sk#20] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#20], Inner, BuildRight - -(24) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(25) ColumnarToRow [codegen id : 1] -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(26) HashAggregate [codegen id : 1] -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Results [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] - -(27) Exchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(28) HashAggregate [codegen id : 2] -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141] -Results [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, DHL,BARIAN AS ship_carriers#142, d_year#16 AS year#143, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118 AS jan_sales#144, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119 AS feb_sales#145, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120 AS mar_sales#146, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121 AS apr_sales#147, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122 AS may_sales#148, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123 AS jun_sales#149, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124 AS jul_sales#150, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125 AS aug_sales#151, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126 AS sep_sales#152, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127 AS oct_sales#153, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128 AS nov_sales#154, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129 AS dec_sales#155, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130 AS jan_net#156, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131 AS feb_net#157, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132 AS mar_net#158, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133 AS apr_net#159, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134 AS may_net#160, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135 AS jun_net#161, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136 AS jul_net#162, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137 AS aug_net#163, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138 AS sep_net#164, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139 AS oct_net#165, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140 AS nov_net#166, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141 AS dec_net#167] - -(29) ReusedExchange [Reuses operator id: 27] -Output [55]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, d_year#174, sum#175, isEmpty#176, sum#177, isEmpty#178, sum#179, isEmpty#180, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222] - -(30) HashAggregate [codegen id : 4] -Input [55]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, d_year#174, sum#175, isEmpty#176, sum#177, isEmpty#178, sum#179, isEmpty#180, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222] -Keys [7]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, d_year#174] -Functions [24]: [sum(CASE WHEN (d_moy#223 = 1) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 2) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 3) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 4) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 5) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 6) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 7) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 8) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 9) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 10) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 11) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 12) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 1) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 2) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 3) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 4) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 5) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 6) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 7) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 8) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 9) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 10) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 11) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 12) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#223 = 1) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#227, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#228, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#229, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#230, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#231, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#232, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#233, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#234, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#235, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#236, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#237, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#238, sum(CASE WHEN (d_moy#223 = 1) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#239, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#240, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#241, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#242, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#243, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#244, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#245, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#246, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#247, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#248, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#249, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#250] -Results [32]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, DHL,BARIAN AS ship_carriers#251, d_year#174 AS year#252, sum(CASE WHEN (d_moy#223 = 1) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#227 AS jan_sales#253, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#228 AS feb_sales#254, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#229 AS mar_sales#255, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#230 AS apr_sales#256, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#231 AS may_sales#257, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#232 AS jun_sales#258, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#233 AS jul_sales#259, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#234 AS aug_sales#260, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#235 AS sep_sales#261, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#236 AS oct_sales#262, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#237 AS nov_sales#263, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#238 AS dec_sales#264, sum(CASE WHEN (d_moy#223 = 1) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#239 AS jan_net#265, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#240 AS feb_net#266, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#241 AS mar_net#267, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#242 AS apr_net#268, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#243 AS may_net#269, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#244 AS jun_net#270, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#245 AS jul_net#271, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#246 AS aug_net#272, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#247 AS sep_net#273, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#248 AS oct_net#274, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#249 AS nov_net#275, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#250 AS dec_net#276] - -(31) Union - -(32) HashAggregate [codegen id : 5] -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#144, feb_sales#145, mar_sales#146, apr_sales#147, may_sales#148, jun_sales#149, jul_sales#150, aug_sales#151, sep_sales#152, oct_sales#153, nov_sales#154, dec_sales#155, jan_net#156, feb_net#157, mar_net#158, apr_net#159, may_net#160, jun_net#161, jul_net#162, aug_net#163, sep_net#164, oct_net#165, nov_net#166, dec_net#167] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [partial_sum(jan_sales#144), partial_sum(feb_sales#145), partial_sum(mar_sales#146), partial_sum(apr_sales#147), partial_sum(may_sales#148), partial_sum(jun_sales#149), partial_sum(jul_sales#150), partial_sum(aug_sales#151), partial_sum(sep_sales#152), partial_sum(oct_sales#153), partial_sum(nov_sales#154), partial_sum(dec_sales#155), partial_sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#156), partial_sum(feb_net#157), partial_sum(mar_net#158), partial_sum(apr_net#159), partial_sum(may_net#160), partial_sum(jun_net#161), partial_sum(jul_net#162), partial_sum(aug_net#163), partial_sum(sep_net#164), partial_sum(oct_net#165), partial_sum(nov_net#166), partial_sum(dec_net#167)] -Aggregate Attributes [72]: [sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288, sum#289, isEmpty#290, sum#291, isEmpty#292, sum#293, isEmpty#294, sum#295, isEmpty#296, sum#297, isEmpty#298, sum#299, isEmpty#300, sum#301, isEmpty#302, sum#303, isEmpty#304, sum#305, isEmpty#306, sum#307, isEmpty#308, sum#309, isEmpty#310, sum#311, isEmpty#312, sum#313, isEmpty#314, sum#315, isEmpty#316, sum#317, isEmpty#318, sum#319, isEmpty#320, sum#321, isEmpty#322, sum#323, isEmpty#324, sum#325, isEmpty#326, sum#327, isEmpty#328, sum#329, isEmpty#330, sum#331, isEmpty#332, sum#333, isEmpty#334, sum#335, isEmpty#336, sum#337, isEmpty#338, sum#339, isEmpty#340, sum#341, isEmpty#342, sum#343, isEmpty#344, sum#345, isEmpty#346, sum#347, isEmpty#348] -Results [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420] - -(33) Exchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(34) HashAggregate [codegen id : 6] -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [sum(jan_sales#144), sum(feb_sales#145), sum(mar_sales#146), sum(apr_sales#147), sum(may_sales#148), sum(jun_sales#149), sum(jul_sales#150), sum(aug_sales#151), sum(sep_sales#152), sum(oct_sales#153), sum(nov_sales#154), sum(dec_sales#155), sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#156), sum(feb_net#157), sum(mar_net#158), sum(apr_net#159), sum(may_net#160), sum(jun_net#161), sum(jul_net#162), sum(aug_net#163), sum(sep_net#164), sum(oct_net#165), sum(nov_net#166), sum(dec_net#167)] -Aggregate Attributes [36]: [sum(jan_sales#144)#421, sum(feb_sales#145)#422, sum(mar_sales#146)#423, sum(apr_sales#147)#424, sum(may_sales#148)#425, sum(jun_sales#149)#426, sum(jul_sales#150)#427, sum(aug_sales#151)#428, sum(sep_sales#152)#429, sum(oct_sales#153)#430, sum(nov_sales#154)#431, sum(dec_sales#155)#432, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#433, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#434, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#435, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#436, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#437, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#438, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#439, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#440, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#441, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#442, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#443, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#444, sum(jan_net#156)#445, sum(feb_net#157)#446, sum(mar_net#158)#447, sum(apr_net#159)#448, sum(may_net#160)#449, sum(jun_net#161)#450, sum(jul_net#162)#451, sum(aug_net#163)#452, sum(sep_net#164)#453, sum(oct_net#165)#454, sum(nov_net#166)#455, sum(dec_net#167)#456] -Results [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum(jan_sales#144)#421 AS jan_sales#457, sum(feb_sales#145)#422 AS feb_sales#458, sum(mar_sales#146)#423 AS mar_sales#459, sum(apr_sales#147)#424 AS apr_sales#460, sum(may_sales#148)#425 AS may_sales#461, sum(jun_sales#149)#426 AS jun_sales#462, sum(jul_sales#150)#427 AS jul_sales#463, sum(aug_sales#151)#428 AS aug_sales#464, sum(sep_sales#152)#429 AS sep_sales#465, sum(oct_sales#153)#430 AS oct_sales#466, sum(nov_sales#154)#431 AS nov_sales#467, sum(dec_sales#155)#432 AS dec_sales#468, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#433 AS jan_sales_per_sq_foot#469, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#434 AS feb_sales_per_sq_foot#470, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#435 AS mar_sales_per_sq_foot#471, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#436 AS apr_sales_per_sq_foot#472, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#437 AS may_sales_per_sq_foot#473, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#438 AS jun_sales_per_sq_foot#474, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#439 AS jul_sales_per_sq_foot#475, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#440 AS aug_sales_per_sq_foot#476, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#441 AS sep_sales_per_sq_foot#477, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#442 AS oct_sales_per_sq_foot#478, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#443 AS nov_sales_per_sq_foot#479, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#444 AS dec_sales_per_sq_foot#480, sum(jan_net#156)#445 AS jan_net#481, sum(feb_net#157)#446 AS feb_net#482, sum(mar_net#158)#447 AS mar_net#483, sum(apr_net#159)#448 AS apr_net#484, sum(may_net#160)#449 AS may_net#485, sum(jun_net#161)#450 AS jun_net#486, sum(jul_net#162)#451 AS jul_net#487, sum(aug_net#163)#452 AS aug_net#488, sum(sep_net#164)#453 AS sep_net#489, sum(oct_net#165)#454 AS oct_net#490, sum(nov_net#166)#455 AS nov_net#491, sum(dec_net#167)#456 AS dec_net#492] - -(35) TakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#457, feb_sales#458, mar_sales#459, apr_sales#460, may_sales#461, jun_sales#462, jul_sales#463, aug_sales#464, sep_sales#465, oct_sales#466, nov_sales#467, dec_sales#468, jan_sales_per_sq_foot#469, feb_sales_per_sq_foot#470, mar_sales_per_sq_foot#471, apr_sales_per_sq_foot#472, may_sales_per_sq_foot#473, jun_sales_per_sq_foot#474, jul_sales_per_sq_foot#475, aug_sales_per_sq_foot#476, sep_sales_per_sq_foot#477, oct_sales_per_sq_foot#478, nov_sales_per_sq_foot#479, dec_sales_per_sq_foot#480, jan_net#481, feb_net#482, mar_net#483, apr_net#484, may_net#485, jun_net#486, jul_net#487, aug_net#488, sep_net#489, oct_net#490, nov_net#491, dec_net#492] -Arguments: 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#457, feb_sales#458, mar_sales#459, apr_sales#460, may_sales#461, jun_sales#462, jul_sales#463, aug_sales#464, sep_sales#465, oct_sales#466, nov_sales#467, dec_sales#468, jan_sales_per_sq_foot#469, feb_sales_per_sq_foot#470, mar_sales_per_sq_foot#471, apr_sales_per_sq_foot#472, may_sales_per_sq_foot#473, jun_sales_per_sq_foot#474, jul_sales_per_sq_foot#475, aug_sales_per_sq_foot#476, sep_sales_per_sq_foot#477, oct_sales_per_sq_foot#478, nov_sales_per_sq_foot#479, dec_sales_per_sq_foot#480, jan_net#481, feb_net#482, mar_net#483, apr_net#484, may_net#485, jun_net#486, jul_net#487, aug_net#488, sep_net#489, oct_net#490, nov_net#491, dec_net#492] +(8) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] + +(12) CometBroadcastHashJoin +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight + +(13) CometProject +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(14) CometNativeScan parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#19, t_time#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) + +(16) CometProject +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] + +(17) CometBroadcastExchange +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] + +(18) CometBroadcastHashJoin +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight + +(19) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(20) CometNativeScan parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_carrier#22, 20)) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) + +(22) CometProject +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] + +(23) CometBroadcastExchange +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] + +(24) CometBroadcastHashJoin +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(26) CometHashAggregate +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(27) CometExchange +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometHashAggregate +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] +ReadSchema: struct + +(30) CometFilter +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(32) CometBroadcastHashJoin +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight + +(33) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(34) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] + +(35) CometBroadcastHashJoin +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight + +(36) CometProject +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(37) ReusedExchange [Reuses operator id: 17] +Output [1]: [t_time_sk#88] + +(38) CometBroadcastHashJoin +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight + +(39) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(40) ReusedExchange [Reuses operator id: 23] +Output [1]: [sm_ship_mode_sk#89] + +(41) CometBroadcastHashJoin +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight + +(42) CometProject +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(43) CometHashAggregate +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(44) CometExchange +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(45) CometHashAggregate +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(46) CometUnion +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] + +(47) CometHashAggregate +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] + +(48) CometExchange +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(49) CometHashAggregate +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] + +(50) CometTakeOrderedAndProject +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] + +(51) CometColumnarToRow [codegen id : 1] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/extended.txt new file mode 100644 index 0000000000..3dee6b4b4e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/extended.txt @@ -0,0 +1,64 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.time_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.ship_mode + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.ship_mode + +Comet accelerated 61 out of 61 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/simplified.txt index c9f9d8272e..64ef752d91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/simplified.txt @@ -1,45 +1,53 @@ -TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - WholeStageCodegen (6) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - WholeStageCodegen (5) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] - CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [t_time_sk] #5 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_time] - CometBroadcastExchange [sm_ship_mode_sk] #6 - CometProject [sm_ship_mode_sk] - CometFilter [sm_ship_mode_sk,sm_carrier] - CometNativeScan: `spark_catalog`.`default`.`ship_mode` [sm_ship_mode_sk,sm_carrier] - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 + CometHashAggregate [jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 + CometHashAggregate [d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [t_time_sk] #5 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_time] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_time] + CometBroadcastExchange [sm_ship_mode_sk] #6 + CometProject [sm_ship_mode_sk] + CometFilter [sm_ship_mode_sk,sm_carrier] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 + CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/explain.txt index afa4e7c07c..40795257cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/explain.txt @@ -1,48 +1,58 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * Sort (27) - +- Exchange (26) - +- WindowGroupLimit (25) - +- * Sort (24) - +- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`item` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (34) ++- * Filter (33) + +- Window (32) + +- WindowGroupLimit (31) + +- * CometColumnarToRow (30) + +- CometSort (29) + +- CometColumnarExchange (28) + +- WindowGroupLimit (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.item (15) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] @@ -65,102 +75,115 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#12, 16)) AS s_store_id#13] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] +Right output [2]: [s_store_sk#11, s_store_id#13] Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] - -(14) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) - -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight - -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] - -(19) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] - -(20) ColumnarToRow [codegen id : 1] -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] - -(21) HashAggregate [codegen id : 1] -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) + +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#15, 50)) AS i_brand#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#16, 50)) AS i_class#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#18, 50)) AS i_product_name#22] + +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] + +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight + +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] + +(21) CometExpand +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] + +(22) CometHashAggregate +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -(22) Exchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(23) CometExchange +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 2] -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(24) CometHashAggregate +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31] -Results [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31 AS sumsales#32] -(24) Sort [codegen id : 2] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 +(25) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] + +(26) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] + +(27) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Partial -(25) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18], [sumsales#32 DESC NULLS LAST], rank(sumsales#32), 100, Partial +(28) CometColumnarExchange +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(26) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(29) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] -(27) Sort [codegen id : 3] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 +(30) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] -(28) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18], [sumsales#32 DESC NULLS LAST], rank(sumsales#32), 100, Final +(31) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Final -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [rank(sumsales#32) windowspecdefinition(i_category#18, sumsales#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [i_category#18], [sumsales#32 DESC NULLS LAST] +(32) Window +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] -(30) Filter [codegen id : 4] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Condition : (rk#33 <= 100) +(33) Filter [codegen id : 3] +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#32 ASC NULLS FIRST, rk#33 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] +(34) TakeOrderedAndProject +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/extended.txt new file mode 100644 index 0000000000..36820cf87f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/extended.txt @@ -0,0 +1,36 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 27 out of 32 eligible operators (84%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/simplified.txt index 42308b5b6e..dc17b92f12 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/simplified.txt @@ -1,39 +1,40 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (4) + WholeStageCodegen (3) Filter [rk] InputAdapter Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (3) - Sort [i_category,sumsales] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (2) - Sort [i_category,sumsales] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] - CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [spark_grouping_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/explain.txt index deb0db7cd8..c3d6784b3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) +TakeOrderedAndProject (43) ++- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) : :- * HashAggregate (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- * ColumnarToRow (26) + : : +- * CometColumnarToRow (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) : : :- CometProject (20) @@ -17,43 +17,51 @@ TakeOrderedAndProject (42) : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : : : +- CometNativeScan parquet spark_catalog.default.store (9) : : : +- CometBroadcastExchange (18) : : : +- CometProject (17) : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (30) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.customer_address (21) + : +- BroadcastExchange (34) + : +- * CometColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometNativeScan parquet spark_catalog.default.customer (30) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometFilter (38) + +- CometNativeScan parquet spark_catalog.default.customer_address (37) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_dom#12] -Arguments: [d_date_sk#10, d_year#11, d_dom#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_dom#12] @@ -76,9 +84,12 @@ Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13, s_city#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#13, s_city#14] @@ -101,9 +112,12 @@ Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] @@ -126,9 +140,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(21) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct (22) CometFilter Input [2]: [ca_address_sk#18, ca_city#19] @@ -147,78 +164,86 @@ Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] -(26) ColumnarToRow [codegen id : 1] -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] - -(27) HashAggregate [codegen id : 1] +(26) CometHashAggregate Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum#20, sum#21, sum#22] -Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -(28) Exchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(27) CometExchange +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(29) HashAggregate [codegen id : 4] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] +(28) CometColumnarToRow [codegen id : 3] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] + +(29) HashAggregate [codegen id : 3] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#26, sum(UnscaledValue(ss_ext_list_price#7))#27, sum(UnscaledValue(ss_ext_tax#8))#28] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#26,17,2) AS extended_price#30, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#27,17,2) AS list_price#31, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#28,17,2) AS extended_tax#32] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#23, sum(UnscaledValue(ss_ext_list_price#7))#24, sum(UnscaledValue(ss_ext_tax#8))#25] +Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#26, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#23,17,2) AS extended_price#27, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#24,17,2) AS list_price#28, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#25,17,2) AS extended_tax#29] -(30) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Arguments: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(30) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (31) CometFilter -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#34)) +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_current_addr_sk#31)) + +(32) CometProject +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35], [c_customer_sk#30, c_current_addr_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#32, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#33, 30)) AS c_last_name#35] -(32) ColumnarToRow [codegen id : 2] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] -(33) BroadcastExchange -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(34) BroadcastExchange +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(34) BroadcastHashJoin [codegen id : 4] +(35) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#33] +Right keys [1]: [c_customer_sk#30] Join type: Inner Join condition: None -(35) Project [codegen id : 4] -Output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(36) Project [codegen id : 3] +Output [8]: [ss_ticket_number#5, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_current_addr_sk#31, c_first_name#34, c_last_name#35] +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#37, ca_city#38] -Arguments: [ca_address_sk#37, ca_city#38] +(37) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#36, ca_city#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct -(37) CometFilter -Input [2]: [ca_address_sk#37, ca_city#38] -Condition : (isnotnull(ca_address_sk#37) AND isnotnull(ca_city#38)) +(38) CometFilter +Input [2]: [ca_address_sk#36, ca_city#37] +Condition : (isnotnull(ca_address_sk#36) AND isnotnull(ca_city#37)) -(38) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#37, ca_city#38] +(39) CometColumnarToRow [codegen id : 2] +Input [2]: [ca_address_sk#36, ca_city#37] -(39) BroadcastExchange -Input [2]: [ca_address_sk#37, ca_city#38] +(40) BroadcastExchange +Input [2]: [ca_address_sk#36, ca_city#37] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(40) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#37] +(41) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [c_current_addr_sk#31] +Right keys [1]: [ca_address_sk#36] Join type: Inner -Join condition: NOT (ca_city#38 = bought_city#29) +Join condition: NOT (ca_city#37 = bought_city#26) -(41) Project [codegen id : 4] -Output [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36, ca_address_sk#37, ca_city#38] +(42) Project [codegen id : 3] +Output [8]: [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] +Input [10]: [ss_ticket_number#5, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_current_addr_sk#31, c_first_name#34, c_last_name#35, ca_address_sk#36, ca_city#37] -(42) TakeOrderedAndProject -Input [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Arguments: 100, [c_last_name#36 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +(43) TakeOrderedAndProject +Input [8]: [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] +Arguments: 100, [c_last_name#35 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/extended.txt new file mode 100644 index 0000000000..2495778b5f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/extended.txt @@ -0,0 +1,45 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_ticket_number#1, ss_customer_sk#2, ca_city#3 AS bought_city#4, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#6,17,2) AS extended_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#8))#9,17,2) AS list_price#10, MakeDecimal(sum(UnscaledValue(ss_ext_tax#11))#12,17,2) AS extended_tax#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 32 out of 40 eligible operators (80%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/simplified.txt index 92eae58c2f..e7d107d642 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/simplified.txt @@ -1,52 +1,51 @@ TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] BroadcastHashJoin [ss_customer_sk,c_customer_sk] HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #3 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #5 - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometColumnarToRow + InputAdapter + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow + WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/explain.txt index 709357a617..f0d5e078c8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/explain.txt @@ -1,60 +1,76 @@ == Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin LeftAnti BuildRight (21) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (46) ++- * HashAggregate (45) + +- * CometColumnarToRow (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (39) + +- * CometColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (35) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6, d_year#7, d_moy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] @@ -86,12 +102,15 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#11] @@ -105,7 +124,7 @@ Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#9] (18) BroadcastExchange @@ -118,92 +137,128 @@ Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#12] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] +ReadSchema: struct -(21) BroadcastHashJoin [codegen id : 5] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#14] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [cs_ship_customer_sk#12], [cs_ship_customer_sk#12] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#12] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] Right keys [1]: [cs_ship_customer_sk#12] Join type: LeftAnti Join condition: None -(22) Project [codegen id : 5] +(27) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13, ca_state#14] +(28) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(24) CometFilter -Input [2]: [ca_address_sk#13, ca_state#14] -Condition : (ca_state#14 IN (KY,GA,NM) AND isnotnull(ca_address_sk#13)) +(29) CometFilter +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#16, 2)) IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) -(25) CometProject -Input [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13], [ca_address_sk#13] +(30) CometProject +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] -(26) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#13] +(31) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#15] -(27) BroadcastExchange -Input [1]: [ca_address_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(32) BroadcastExchange +Input [1]: [ca_address_sk#15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(28) BroadcastHashJoin [codegen id : 5] +(33) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#13] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(29) Project [codegen id : 5] +(34) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#13] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(35) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(31) CometFilter -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Condition : isnotnull(cd_demo_sk#15) +(36) CometFilter +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Condition : isnotnull(cd_demo_sk#17) -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(37) CometProject +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Arguments: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26], [cd_demo_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#18, 1)) AS cd_gender#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#25, cd_purchase_estimate#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#22, 10)) AS cd_credit_rating#26] -(33) BroadcastExchange -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(38) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] -(34) BroadcastHashJoin [codegen id : 5] +(39) BroadcastExchange +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#15] +Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: None -(35) Project [codegen id : 5] -Output [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(41) Project [codegen id : 5] +Output [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] -(36) HashAggregate [codegen id : 5] -Input [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(42) HashAggregate [codegen id : 5] +Input [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#21] -Results [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] +Aggregate Attributes [1]: [count#27] +Results [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] + +(43) CometColumnarExchange +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(37) Exchange -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Arguments: hashpartitioning(cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(44) CometColumnarToRow [codegen id : 6] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] -(38) HashAggregate [codegen id : 6] -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +(45) HashAggregate [codegen id : 6] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#23] -Results [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, count(1)#23 AS cnt1#24, cd_purchase_estimate#19, count(1)#23 AS cnt2#25, cd_credit_rating#20, count(1)#23 AS cnt3#26] +Aggregate Attributes [1]: [count(1)#29] +Results [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#29 AS cnt1#30, cd_purchase_estimate#21, count(1)#29 AS cnt2#31, cd_credit_rating#26, count(1)#29 AS cnt3#32] -(39) TakeOrderedAndProject -Input [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] -Arguments: 100, [cd_gender#16 ASC NULLS FIRST, cd_marital_status#17 ASC NULLS FIRST, cd_education_status#18 ASC NULLS FIRST, cd_purchase_estimate#19 ASC NULLS FIRST, cd_credit_rating#20 ASC NULLS FIRST], [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] +(46) TakeOrderedAndProject +Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] +Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#26 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/extended.txt new file mode 100644 index 0000000000..0fa566e7ff --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/extended.txt @@ -0,0 +1,54 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 46 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/simplified.txt index f8b77163ae..69d8e49a63 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/simplified.txt @@ -1,53 +1,62 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/explain.txt index 2802968d2e..b53beef96e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/explain.txt @@ -1,51 +1,59 @@ == Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.promotion (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -64,9 +72,12 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#13, d_year#14] @@ -89,74 +100,80 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometNativeScan: `spark_catalog`.`default`.`item` +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] +Right output [2]: [i_item_sk#15, i_item_id#17] Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [ss_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(26) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(27) HashAggregate [codegen id : 1] -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] + +(21) CometNativeScan parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#19, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#20, 1)) = N)) AND isnotnull(p_promo_sk#18)) + +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] + +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight + +(26) CometProject +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] + +(27) CometHashAggregate +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -(28) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(29) HashAggregate [codegen id : 2] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#36, avg(UnscaledValue(ss_list_price#5))#37, avg(UnscaledValue(ss_coupon_amt#7))#38, avg(UnscaledValue(ss_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(ss_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(ss_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(ss_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(ss_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] -(30) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/extended.txt new file mode 100644 index 0000000000..4848e7820f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/simplified.txt index 999cdc29ab..0867ca71bb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/simplified.txt @@ -1,34 +1,33 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (2) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] - CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #5 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] + CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/explain.txt index eef9f1004c..6399e44ea6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/explain.txt @@ -1,62 +1,73 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Expand (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- BroadcastExchange (34) - +- * BroadcastHashJoin LeftSemi BuildRight (33) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (10) - +- BroadcastExchange (32) - +- * Project (31) - +- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * Sort (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * ColumnarToRow (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- ReusedExchange (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (49) ++- * Project (48) + +- Window (47) + +- * CometColumnarToRow (46) + +- CometSort (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometColumnarExchange (41) + +- * HashAggregate (40) + +- * Expand (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (36) + +- * Project (35) + +- * BroadcastHashJoin LeftSemi BuildRight (34) + :- * CometColumnarToRow (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.store (10) + +- BroadcastExchange (33) + +- * Project (32) + +- * Filter (31) + +- Window (30) + +- WindowGroupLimit (29) + +- * Sort (28) + +- * HashAggregate (27) + +- * CometColumnarToRow (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometFilter (14) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + +- ReusedExchange (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -79,165 +90,187 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) ColumnarToRow [codegen id : 5] +(9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometNativeScan: `spark_catalog`.`default`.`store` +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: [s_store_sk#6, s_county#7, s_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] Condition : isnotnull(s_store_sk#6) -(12) ColumnarToRow [codegen id : 4] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Arguments: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#12, s_state#13] Condition : isnotnull(s_store_sk#12) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#14] -(18) CometBroadcastHashJoin +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] +Right output [2]: [s_store_sk#12, s_state#14] Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight -(23) ColumnarToRow [codegen id : 1] -Input [2]: [ss_net_profit#10, s_state#13] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] -(24) HashAggregate [codegen id : 1] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] -(25) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(26) CometColumnarToRow [codegen id : 1] +Input [2]: [s_state#14, sum#16] -(26) HashAggregate [codegen id : 2] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] +(27) HashAggregate [codegen id : 1] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] +Results [3]: [s_state#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#14] -(27) Sort [codegen id : 2] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 +(28) Sort [codegen id : 1] +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final -(29) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] +(30) Window +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [rank(_w0#18) windowspecdefinition(s_state#14, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#14], [_w0#18 DESC NULLS LAST] -(30) Filter [codegen id : 3] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] Condition : (ranking#19 <= 5) -(31) Project [codegen id : 3] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] -(32) BroadcastExchange -Input [1]: [s_state#13] +(33) BroadcastExchange +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] -(33) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2))] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None -(34) BroadcastExchange +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2)) AS s_state#20] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 5] +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(36) Project [codegen id : 5] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_state#20, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#20] -(37) Expand [codegen id : 5] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] +(39) Expand [codegen id : 4] +Input [3]: [ss_net_profit#2, s_state#20, s_county#7] +Arguments: [[ss_net_profit#2, s_state#20, s_county#7, 0], [ss_net_profit#2, s_state#20, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#21, s_county#22, spark_grouping_id#23] -(38) HashAggregate [codegen id : 5] -Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] +(40) HashAggregate [codegen id : 4] +Input [4]: [ss_net_profit#2, s_state#21, s_county#22, spark_grouping_id#23] +Keys [3]: [s_state#21, s_county#22, spark_grouping_id#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#23] -Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Aggregate Attributes [1]: [sum#24] +Results [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] + +(41) CometColumnarExchange +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] +Arguments: hashpartitioning(s_state#21, s_county#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(39) Exchange -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(42) CometColumnarToRow [codegen id : 5] +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] -(40) HashAggregate [codegen id : 6] -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] +(43) HashAggregate [codegen id : 5] +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] +Keys [3]: [s_state#21, s_county#22, spark_grouping_id#23] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#26] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#26,17,2) AS total_sum#27, s_state#21, s_county#22, (cast((shiftright(spark_grouping_id#23, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint)) AS lochierarchy#28, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#26,17,2) AS _w0#29, (cast((shiftright(spark_grouping_id#23, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint)) AS _w1#30, CASE WHEN (cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint) = 0) THEN s_state#21 END AS _w2#31] + +(44) CometColumnarExchange +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: hashpartitioning(_w1#30, _w2#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(41) Exchange -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(45) CometSort +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31], [_w1#30 ASC NULLS FIRST, _w2#31 ASC NULLS FIRST, _w0#29 DESC NULLS LAST] -(42) Sort [codegen id : 7] -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST], false, 0 +(46) CometColumnarToRow [codegen id : 6] +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] -(43) Window -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] +(47) Window +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: [rank(_w0#29) windowspecdefinition(_w1#30, _w2#31, _w0#29 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#32], [_w1#30, _w2#31], [_w0#29 DESC NULLS LAST] -(44) Project [codegen id : 8] -Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] +(48) Project [codegen id : 7] +Output [5]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] +Input [8]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31, rank_within_parent#32] -(45) TakeOrderedAndProject -Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +(49) TakeOrderedAndProject +Input [5]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#21 END ASC NULLS FIRST, rank_within_parent#32 ASC NULLS FIRST], [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/extended.txt new file mode 100644 index 0000000000..7d068c1e16 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/extended.txt @@ -0,0 +1,54 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- Sort + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_state#1, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#3,17,2) AS _w0#4, s_state#1)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 29 out of 47 eligible operators (61%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/simplified.txt index fdf0219cc5..9b341eb809 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/simplified.txt @@ -1,63 +1,65 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (8) + WholeStageCodegen (7) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (7) - Sort [_w1,_w2,_w0] + WholeStageCodegen (6) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (6) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (2) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] - InputAdapter - Exchange [s_state] #6 - WholeStageCodegen (1) - HashAggregate [s_state,ss_net_profit] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #7 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - ReusedExchange [d_date_sk] #3 + CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + Sort [s_state,_w0] + HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_state] #6 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #7 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/explain.txt index 117ac1349d..c621702992 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* ColumnarToRow (38) +* CometColumnarToRow (38) +- CometSort (37) +- CometColumnarExchange (36) +- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * ColumnarToRow (32) + +- * CometColumnarToRow (34) + +- CometExchange (33) + +- CometHashAggregate (32) +- CometProject (31) +- CometBroadcastHashJoin (30) :- CometProject (25) @@ -13,35 +13,38 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : +- CometNativeScan parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) : : :- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (5) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (5) : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (7) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (13) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (13) : : +- ReusedExchange (15) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (19) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (18) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (20) +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (26) + +- CometNativeScan parquet spark_catalog.default.time_dim (26) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] @@ -49,153 +52,169 @@ Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull( (3) CometProject Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3], [i_item_sk#1, i_brand_id#2, i_brand#3] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#3, 50)) AS i_brand#5] (4) CometBroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3] +Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(5) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Arguments: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +(5) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] +ReadSchema: struct (6) CometFilter -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) +Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) -(7) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] +(7) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (8) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) (9) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (10) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (11) CometBroadcastHashJoin -Left output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (12) CometProject -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] -Arguments: [ext_price#12, sold_item_sk#13, time_sk#14], [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] -(13) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(13) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] +ReadSchema: struct (14) CometFilter -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) (15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#20] (16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (17) CometProject -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [ext_price#20, sold_item_sk#21, time_sk#22], [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] -(18) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Arguments: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(18) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] +ReadSchema: struct (19) CometFilter -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) (20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#27] +Output [1]: [d_date_sk#28] (21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (22) CometProject -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] -Arguments: [ext_price#28, sold_item_sk#29, time_sk#30], [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] (23) CometUnion -Child 0 Input [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Child 1 Input [3]: [ext_price#20, sold_item_sk#21, time_sk#22] -Child 2 Input [3]: [ext_price#28, sold_item_sk#29, time_sk#30] +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] (24) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Right output [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_item_sk#1], [sold_item_sk#13], Inner, BuildLeft +Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft (25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14], [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] -(26) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] +(26) CometNativeScan parquet spark_catalog.default.time_dim +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_time_sk)] +ReadSchema: struct (27) CometFilter -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#35, 20)) = breakfast ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#35, 20)) = dinner )) AND isnotnull(t_time_sk#32)) (28) CometProject -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33], [t_time_sk#31, t_hour#32, t_minute#33] +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] (29) CometBroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] (30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Right output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [time_sk#14], [t_time_sk#31], Inner, BuildRight +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight (31) CometProject -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33], [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] - -(32) ColumnarToRow [codegen id : 1] -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] - -(33) HashAggregate [codegen id : 1] -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum#35] -Results [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] - -(34) Exchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(35) HashAggregate [codegen id : 2] -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#12))#37] -Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#3 AS brand#39, t_hour#32, t_minute#33, MakeDecimal(sum(UnscaledValue(ext_price#12))#37,17,2) AS ext_price#40] +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] + +(32) CometHashAggregate +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] + +(33) CometExchange +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] + +(35) HashAggregate [codegen id : 1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#13))#37] +Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#5 AS brand#39, t_hour#33, t_minute#34, MakeDecimal(sum(UnscaledValue(ext_price#13))#37,17,2) AS ext_price#40] (36) CometColumnarExchange -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] Arguments: rangepartitioning(ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (37) CometSort -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40], [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST] +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] +Arguments: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40], [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST] -(38) ColumnarToRow [codegen id : 3] -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] +(38) CometColumnarToRow [codegen id : 2] +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/extended.txt new file mode 100644 index 0000000000..81596a2419 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/extended.txt @@ -0,0 +1,46 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_brand_id#1 AS brand_id#2, i_brand#3 AS brand#4, t_hour#5, t_minute#6, MakeDecimal(sum(UnscaledValue(ext_price#7))#8,17,2) AS ext_price#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometUnion + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.time_dim + +Comet accelerated 41 out of 42 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/simplified.txt index 6fd882ac75..e82ba4fa44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/simplified.txt @@ -1,44 +1,42 @@ -WholeStageCodegen (3) - ColumnarToRow +WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometSort [brand_id,brand,t_hour,t_minute,ext_price] CometColumnarExchange [ext_price,brand_id] #1 - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,t_hour,t_minute] #2 - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] - CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] - CometProject [i_brand_id,i_brand,ext_price,time_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometUnion [ext_price,sold_item_sk,time_sk] - CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] - CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 - CometProject [t_time_sk,t_hour,t_minute] - CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute,t_meal_time] + CometColumnarToRow + InputAdapter + CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 + CometHashAggregate [ext_price] [i_brand,i_brand_id,t_hour,t_minute,sum] + CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] + CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] + CometProject [i_brand_id,i_brand,ext_price,time_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometUnion [ext_price,sold_item_sk,time_sk] + CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 + CometProject [t_time_sk,t_hour,t_minute] + CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/explain.txt index 2ef4c4fa34..18e85969ff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/explain.txt @@ -1,79 +1,89 @@ == Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometHashAggregate (60) - +- CometExchange (59) - +- CometHashAggregate (58) - +- CometProject (57) - +- CometSortMergeJoin (56) - :- CometSort (50) - : +- CometExchange (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometBroadcastHashJoin (37) - : : : :- CometProject (33) - : : : : +- CometBroadcastHashJoin (32) - : : : : :- CometProject (27) - : : : : : +- CometBroadcastHashJoin (26) - : : : : : :- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) : : : : : : : :- CometProject (12) : : : : : : : : +- CometBroadcastHashJoin (11) : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (3) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : : : : : : : +- ReusedExchange (13) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (16) - : : : : : +- CometBroadcastExchange (25) - : : : : : +- CometProject (24) - : : : : : +- CometFilter (23) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (22) - : : : : +- CometBroadcastExchange (31) - : : : : +- CometProject (30) - : : : : +- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (28) - : : : +- CometBroadcastExchange (36) - : : : +- CometFilter (35) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (44) - +- CometSort (55) - +- CometExchange (54) - +- CometProject (53) - +- CometFilter (52) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (51) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`inventory` +(3) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#12)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] @@ -92,9 +102,12 @@ Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] @@ -113,211 +126,244 @@ Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -(13) ReusedExchange [Reuses operator id: 10] +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_desc#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) -(14) CometBroadcastHashJoin +(15) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] + +(16) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] Right output [2]: [i_item_sk#15, i_item_desc#16] Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight -(15) CometProject +(17) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(16) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17, cd_marital_status#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct -(17) CometFilter +(19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) +Condition : ((isnotnull(cd_marital_status#18) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#18, 1)) = D)) AND isnotnull(cd_demo_sk#17)) -(18) CometProject +(20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] Arguments: [cd_demo_sk#17], [cd_demo_sk#17] -(19) CometBroadcastExchange +(21) CometBroadcastExchange Input [1]: [cd_demo_sk#17] Arguments: [cd_demo_sk#17] -(20) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [1]: [cd_demo_sk#17] Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight -(21) CometProject +(23) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(22) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19, hd_buy_potential#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] +ReadSchema: struct -(23) CometFilter +(25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((isnotnull(hd_buy_potential#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#20, 15)) = >10000 )) AND isnotnull(hd_demo_sk#19)) -(24) CometProject +(26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] Arguments: [hd_demo_sk#19], [hd_demo_sk#19] -(25) CometBroadcastExchange +(27) CometBroadcastExchange Input [1]: [hd_demo_sk#19] Arguments: [hd_demo_sk#19] -(26) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [1]: [hd_demo_sk#19] Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight -(27) CometProject +(29) CometProject Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(28) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] +ReadSchema: struct -(29) CometFilter +(31) CometFilter Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) -(30) CometProject +(32) CometProject Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] -(31) CometBroadcastExchange +(33) CometBroadcastExchange Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] -(32) CometBroadcastHashJoin +(34) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight -(33) CometProject +(35) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(35) CometFilter +(37) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(36) CometBroadcastExchange +(38) CometBroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] Arguments: [d_date_sk#25, d_week_seq#26] -(37) CometBroadcastHashJoin +(39) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] Right output [2]: [d_date_sk#25, d_week_seq#26] Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight -(38) CometProject +(40) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] +ReadSchema: struct -(40) CometFilter +(42) CometFilter Input [2]: [d_date_sk#27, d_date#28] Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(41) CometBroadcastExchange +(43) CometBroadcastExchange Input [2]: [d_date_sk#27, d_date#28] Arguments: [d_date_sk#27, d_date#28] -(42) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] Right output [2]: [d_date_sk#27, d_date#28] Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight -(43) CometProject +(45) CometProject Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(44) CometNativeScan: `spark_catalog`.`default`.`promotion` +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct -(45) CometFilter +(47) CometFilter Input [1]: [p_promo_sk#29] Condition : isnotnull(p_promo_sk#29) -(46) CometBroadcastExchange +(48) CometBroadcastExchange Input [1]: [p_promo_sk#29] Arguments: [p_promo_sk#29] -(47) CometBroadcastHashJoin +(49) CometBroadcastHashJoin Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Right output [1]: [p_promo_sk#29] Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight -(48) CometProject +(50) CometProject Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(49) CometExchange +(51) CometExchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(50) CometSort +(52) CometSort Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(51) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct -(52) CometFilter +(54) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(53) CometProject +(55) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] -(54) CometExchange +(56) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(55) CometSort +(57) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] -(56) CometSortMergeJoin +(58) CometSortMergeJoin Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Right output [2]: [cr_item_sk#30, cr_order_number#31] Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter -(57) CometProject +(59) CometProject Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(58) CometHashAggregate +(60) CometHashAggregate Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] -(59) CometExchange +(61) CometExchange Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(60) CometHashAggregate +(62) CometHashAggregate Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] -(61) CometTakeOrderedAndProject +(63) CometTakeOrderedAndProject Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -(62) ColumnarToRow [codegen id : 1] +(64) CometColumnarToRow [codegen id : 1] Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/extended.txt new file mode 100644 index 0000000000..3bf43ef8c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/extended.txt @@ -0,0 +1,66 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_returns + +Comet accelerated 63 out of 63 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/simplified.txt index e70defef63..073bd57a8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] CometProject [w_warehouse_name,i_item_desc,d_week_seq] @@ -28,37 +28,39 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [i_item_sk,i_item_desc] #4 - CometBroadcastExchange [cd_demo_sk] #5 + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #6 + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #7 + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #8 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #9 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #10 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #11 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/explain.txt index 13f2c1319f..b4c3269a2a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,35 +15,43 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] @@ -66,9 +74,12 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#9, s_county#10] @@ -91,13 +102,16 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -134,35 +148,42 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) -(25) CometNativeScan: `spark_catalog`.`default`.`customer` +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (26) CometFilter Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Condition : isnotnull(c_customer_sk#17) -(27) CometBroadcastExchange +(27) CometProject Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] -(28) CometBroadcastHashJoin +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/simplified.txt index 35a931bfdf..145db18334 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [cnt] #1 + CometExchange [cnt] #1 CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] @@ -16,19 +16,20 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/explain.txt index e9b83cae94..c3f16d295b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/explain.txt @@ -1,251 +1,406 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * BroadcastHashJoin Inner BuildRight (33) - : : :- * Filter (17) - : : : +- * HashAggregate (16) - : : : +- Exchange (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- BroadcastExchange (32) - : : +- * HashAggregate (31) - : : +- Exchange (30) - : : +- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (18) - : : : +- ReusedExchange (20) - : : +- CometBroadcastExchange (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * HashAggregate (35) - : +- ReusedExchange (34) - +- BroadcastExchange (42) - +- * HashAggregate (41) - +- ReusedExchange (40) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometNativeScan parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(13) ColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(14) HashAggregate [codegen id : 1] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(15) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(16) HashAggregate [codegen id : 8] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(17) Filter [codegen id : 8] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(18) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Arguments: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] - -(19) CometFilter -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) - -(20) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] - -(21) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Right output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] -Arguments: [c_customer_sk#15], [ss_customer_sk#19], Inner, BuildRight - -(22) CometProject -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] -Arguments: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21], [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] - -(24) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] - -(26) CometBroadcastHashJoin -Left output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Right output [2]: [d_date_sk#22, d_year#23] -Arguments: [ss_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(27) CometProject -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] -Arguments: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23], [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] - -(28) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] - -(29) HashAggregate [codegen id : 2] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(30) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(31) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(32) BroadcastExchange -Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#7] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) + +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] + +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight + +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] + +(17) HashAggregate [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#14] +Results [2]: [c_customer_id#5 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#14,17,2) AS year_total#16] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#15, year_total#16] +Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) + +(19) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)))) + +(21) CometProject +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Arguments: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)) AS c_customer_id#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#23] + +(22) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] + +(23) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23] +Right output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_sk#17], [ss_customer_sk#24], Inner, BuildRight + +(24) CometProject +Input [7]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23, ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] + +(25) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_year#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#27, d_year#28] +Condition : (((isnotnull(d_year#28) AND (d_year#28 = 2002)) AND d_year#28 IN (2001,2002)) AND isnotnull(d_date_sk#27)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_year#28] +Arguments: [d_date_sk#27, d_year#28] + +(28) CometBroadcastHashJoin +Left output [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] +Right output [2]: [d_date_sk#27, d_year#28] +Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight + +(29) CometProject +Input [7]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] + +(30) CometHashAggregate +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#25))] + +(31) CometExchange +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] + +(33) HashAggregate [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [sum(UnscaledValue(ss_net_paid#25))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#25))#14] +Results [4]: [c_customer_id#21 AS customer_id#30, c_first_name#22 AS customer_first_name#31, c_last_name#23 AS customer_last_name#32, MakeDecimal(sum(UnscaledValue(ss_net_paid#25))#14,17,2) AS year_total#33] + +(34) BroadcastExchange +Input [4]: [customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#30] Join type: Inner Join condition: None -(34) ReusedExchange [Reuses operator id: 15] -Output [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#33, sum#34] - -(35) HashAggregate [codegen id : 5] -Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#33, sum#34] -Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#33] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#36] -Results [2]: [c_customer_id#30 AS customer_id#37, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#36,17,2) AS year_total#38] - -(36) Filter [codegen id : 5] -Input [2]: [customer_id#37, year_total#38] -Condition : (isnotnull(year_total#38) AND (year_total#38 > 0.00)) - -(37) BroadcastExchange -Input [2]: [customer_id#37, year_total#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] +(36) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(37) CometFilter +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Condition : (isnotnull(c_customer_sk#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)))) + +(38) CometProject +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Arguments: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40], [c_customer_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)) AS c_customer_id#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#36, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#37, 30)) AS c_last_name#40] + +(39) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_customer_sk#41) + +(41) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] + +(42) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40] +Right output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_sk#34], [ws_bill_customer_sk#41], Inner, BuildRight + +(43) CometProject +Input [7]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40, ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] + +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#44, d_year#45] + +(45) CometBroadcastHashJoin +Left output [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] +Right output [2]: [d_date_sk#44, d_year#45] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(46) CometProject +Input [7]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] + +(47) CometHashAggregate +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#42))] + +(48) CometExchange +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Arguments: hashpartitioning(c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(49) CometColumnarToRow [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] + +(50) HashAggregate [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_net_paid#42))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#42))#47] +Results [2]: [c_customer_id#38 AS customer_id#48, MakeDecimal(sum(UnscaledValue(ws_net_paid#42))#47,17,2) AS year_total#49] + +(51) Filter [codegen id : 2] +Input [2]: [customer_id#48, year_total#49] +Condition : (isnotnull(year_total#49) AND (year_total#49 > 0.00)) + +(52) BroadcastExchange +Input [2]: [customer_id#48, year_total#49] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#37] +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#48] Join type: Inner Join condition: None -(39) Project [codegen id : 8] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#38] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#37, year_total#38] - -(40) ReusedExchange [Reuses operator id: 30] -Output [5]: [c_customer_id#39, c_first_name#40, c_last_name#41, d_year#42, sum#43] - -(41) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#39, c_first_name#40, c_last_name#41, d_year#42, sum#43] -Keys [4]: [c_customer_id#39, c_first_name#40, c_last_name#41, d_year#42] -Functions [1]: [sum(UnscaledValue(ws_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#44))#36] -Results [2]: [c_customer_id#39 AS customer_id#45, MakeDecimal(sum(UnscaledValue(ws_net_paid#44))#36,17,2) AS year_total#46] - -(42) BroadcastExchange -Input [2]: [customer_id#45, year_total#46] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] - -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#45] +(54) Project [codegen id : 4] +Output [7]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49] +Input [8]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, customer_id#48, year_total#49] + +(55) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(56) CometFilter +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Condition : (isnotnull(c_customer_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)))) + +(57) CometProject +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Arguments: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56], [c_customer_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)) AS c_customer_id#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#52, 20)) AS c_first_name#55, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#53, 30)) AS c_last_name#56] + +(58) ReusedExchange [Reuses operator id: 41] +Output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] + +(59) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56] +Right output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#50], [ws_bill_customer_sk#57], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56, ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] + +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#60, d_year#61] + +(62) CometBroadcastHashJoin +Left output [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(63) CometProject +Input [7]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] + +(64) CometHashAggregate +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#58))] + +(65) CometExchange +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] + +(67) HashAggregate [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [sum(UnscaledValue(ws_net_paid#58))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#58))#47] +Results [2]: [c_customer_id#54 AS customer_id#63, MakeDecimal(sum(UnscaledValue(ws_net_paid#58))#47,17,2) AS year_total#64] + +(68) BroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#63] Join type: Inner -Join condition: (CASE WHEN (year_total#38 > 0.00) THEN (year_total#46 / year_total#38) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) +Join condition: (CASE WHEN (year_total#49 > 0.00) THEN (year_total#64 / year_total#49) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#33 / year_total#16) END) -(44) Project [codegen id : 8] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#38, customer_id#45, year_total#46] +(70) Project [codegen id : 4] +Output [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Input [9]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49, customer_id#63, year_total#64] -(45) TakeOrderedAndProject -Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] +(71) TakeOrderedAndProject +Input [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Arguments: 100, [customer_id#30 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST], [customer_id#30, customer_first_name#31, customer_last_name#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/extended.txt new file mode 100644 index 0000000000..696be0ba76 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/extended.txt @@ -0,0 +1,81 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- BroadcastHashJoin + : : :- Filter + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#1 AS customer_id#2, MakeDecimal(sum(UnscaledValue(ss_net_paid#3))#4,17,2) AS year_total#5)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#6 AS customer_id#7, c_first_name#8 AS customer_first_name#9, c_last_name#10 AS customer_last_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#12))#4,17,2) AS year_total#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#14 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ws_net_paid#16))#17,17,2) AS year_total#18)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#19 AS customer_id#20, MakeDecimal(sum(UnscaledValue(ws_net_paid#21))#17,17,2) AS year_total#22)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 60 out of 75 eligible operators (80%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/simplified.txt index 20e538e1bd..c7cb74acaf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] - WholeStageCodegen (8) + WholeStageCodegen (4) Project [customer_id,customer_first_name,customer_last_name] BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] @@ -7,54 +7,76 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] BroadcastHashJoin [customer_id,customer_id] Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (1) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - ColumnarToRow - InputAdapter + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #7 - WholeStageCodegen (5) + WholeStageCodegen (2) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #1 + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) + BroadcastExchange #10 + WholeStageCodegen (3) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #5 + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/explain.txt index e0339b1928..59cd2c3744 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/explain.txt @@ -1,108 +1,154 @@ == Physical Plan == -TakeOrderedAndProject (87) -+- * Project (86) - +- * SortMergeJoin Inner (85) - :- * Sort (45) - : +- Exchange (44) - : +- * Filter (43) - : +- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * ColumnarToRow (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometUnion (35) - : :- CometProject (22) - : : +- CometSortMergeJoin (21) - : : :- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : : +- CometSort (20) - : : +- CometExchange (19) - : : +- CometProject (18) - : : +- CometFilter (17) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : :- CometProject (28) - : : +- CometSortMergeJoin (27) - : : :- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometSort (26) - : : +- ReusedExchange (25) - : +- CometProject (34) - : +- CometSortMergeJoin (33) - : :- CometSort (30) - : : +- ReusedExchange (29) - : +- CometSort (32) - : +- ReusedExchange (31) - +- * Sort (84) - +- Exchange (83) - +- * Filter (82) - +- * HashAggregate (81) - +- Exchange (80) - +- * HashAggregate (79) - +- * ColumnarToRow (78) - +- CometHashAggregate (77) - +- CometExchange (76) - +- CometHashAggregate (75) - +- CometUnion (74) - :- CometProject (61) - : +- CometSortMergeJoin (60) - : :- CometSort (57) - : : +- CometExchange (56) - : : +- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (50) - : : : +- CometBroadcastHashJoin (49) - : : : :- CometFilter (47) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (46) - : : : +- ReusedExchange (48) - : : +- CometBroadcastExchange (53) - : : +- CometFilter (52) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (51) - : +- CometSort (59) - : +- ReusedExchange (58) - :- CometProject (67) - : +- CometSortMergeJoin (66) - : :- CometSort (63) - : : +- ReusedExchange (62) - : +- CometSort (65) - : +- ReusedExchange (64) - +- CometProject (73) - +- CometSortMergeJoin (72) - :- CometSort (69) - : +- ReusedExchange (68) - +- CometSort (71) - +- ReusedExchange (70) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +* CometColumnarToRow (126) ++- CometTakeOrderedAndProject (125) + +- CometProject (124) + +- CometSortMergeJoin (123) + :- CometSort (67) + : +- CometColumnarExchange (66) + : +- * Filter (65) + : +- * HashAggregate (64) + : +- * CometColumnarToRow (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) + +- CometSort (122) + +- CometColumnarExchange (121) + +- * Filter (120) + +- * HashAggregate (119) + +- * CometColumnarToRow (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometHashAggregate (115) + +- CometExchange (114) + +- CometHashAggregate (113) + +- CometUnion (112) + :- CometProject (83) + : +- CometSortMergeJoin (82) + : :- CometSort (79) + : : +- CometExchange (78) + : : +- CometProject (77) + : : +- CometBroadcastHashJoin (76) + : : :- CometProject (72) + : : : +- CometBroadcastHashJoin (71) + : : : :- CometFilter (69) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (68) + : : : +- ReusedExchange (70) + : : +- CometBroadcastExchange (75) + : : +- CometFilter (74) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (73) + : +- CometSort (81) + : +- ReusedExchange (80) + :- CometProject (97) + : +- CometSortMergeJoin (96) + : :- CometSort (93) + : : +- CometExchange (92) + : : +- CometProject (91) + : : +- CometBroadcastHashJoin (90) + : : :- CometProject (88) + : : : +- CometBroadcastHashJoin (87) + : : : :- CometFilter (85) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (84) + : : : +- ReusedExchange (86) + : : +- ReusedExchange (89) + : +- CometSort (95) + : +- ReusedExchange (94) + +- CometProject (111) + +- CometSortMergeJoin (110) + :- CometSort (107) + : +- CometExchange (106) + : +- CometProject (105) + : +- CometBroadcastHashJoin (104) + : :- CometProject (102) + : : +- CometBroadcastHashJoin (101) + : : :- CometFilter (99) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (98) + : : +- ReusedExchange (100) + : +- ReusedExchange (103) + +- CometSort (109) + +- ReusedExchange (108) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((isnotnull(i_category#10) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] @@ -121,9 +167,12 @@ Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#12, d_year#13] @@ -150,9 +199,12 @@ Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREM Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct (17) CometFilter Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] @@ -179,278 +231,463 @@ Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14 Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(23) ReusedExchange [Reuses operator id: 14] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] +(23) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct -(24) CometSort -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] +(24) CometFilter +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) -(25) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] +(25) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(26) CometSort -Input [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33], [sr_ticket_number#31 ASC NULLS FIRST, sr_item_sk#30 ASC NULLS FIRST] +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight -(27) CometSortMergeJoin -Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Right output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#31, sr_item_sk#30], LeftOuter +(27) CometProject +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(28) CometProject -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29, sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35], [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, (ss_quantity#23 - coalesce(sr_return_quantity#32, 0)) AS sales_cnt#34, (ss_ext_sales_price#24 - coalesce(sr_return_amt#33, 0.00)) AS sales_amt#35] +(28) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#31, d_year#32] -(29) ReusedExchange [Reuses operator id: 14] -Output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight -(30) CometSort -Input [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Arguments: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44], [ws_order_number#37 ASC NULLS FIRST, ws_item_sk#36 ASC NULLS FIRST] +(30) CometProject +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -(31) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] +(31) CometExchange +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48], [wr_order_number#46 ASC NULLS FIRST, wr_item_sk#45 ASC NULLS FIRST] - -(33) CometSortMergeJoin -Left output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Right output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [ws_order_number#37, ws_item_sk#36], [wr_order_number#46, wr_item_sk#45], LeftOuter - -(34) CometProject -Input [13]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44, wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50], [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, (ws_quantity#38 - coalesce(wr_return_quantity#47, 0)) AS sales_cnt#49, (ws_ext_sales_price#39 - coalesce(wr_return_amt#48, 0.00)) AS sales_amt#50] - -(35) CometUnion +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] + +(33) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) + +(35) CometProject +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] + +(36) CometExchange +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(37) CometSort +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] + +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter + +(39) CometProject +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] + +(40) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(41) CometFilter +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) + +(42) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight + +(44) CometProject +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#50, d_year#51] + +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight + +(47) CometProject +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] + +(48) CometExchange +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(49) CometSort +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] + +(50) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) + +(52) CometProject +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] + +(53) CometExchange +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(54) CometSort +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter + +(56) CometProject +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] + +(57) CometUnion Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] -(36) CometHashAggregate +(58) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -(37) CometExchange +(59) CometExchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(38) CometHashAggregate +(60) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -(39) ColumnarToRow [codegen id : 1] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] - -(40) HashAggregate [codegen id : 1] +(61) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#53, sum#54] -(41) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#53, sum#54] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(63) CometColumnarToRow [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] -(42) HashAggregate [codegen id : 2] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#53, sum#54] +(64) HashAggregate [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#55, sum(UnscaledValue(sales_amt#20))#56] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#55 AS sales_cnt#57, MakeDecimal(sum(UnscaledValue(sales_amt#20))#56,18,2) AS sales_amt#58] - -(43) Filter [codegen id : 2] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58] -Condition : isnotnull(sales_cnt#57) - -(44) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(45) Sort [codegen id : 3] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 - -(46) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] - -(47) CometFilter -Input [5]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] -Condition : isnotnull(cs_item_sk#59) - -(48) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#64, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] - -(49) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] -Right output [5]: [i_item_sk#64, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Arguments: [cs_item_sk#59], [i_item_sk#64], Inner, BuildRight - -(50) CometProject -Input [10]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_item_sk#64, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68], [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] - -(51) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69, d_year#70] - -(52) CometFilter -Input [2]: [d_date_sk#69, d_year#70] -Condition : ((isnotnull(d_year#70) AND (d_year#70 = 2001)) AND isnotnull(d_date_sk#69)) - -(53) CometBroadcastExchange -Input [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69, d_year#70] - -(54) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Right output [2]: [d_date_sk#69, d_year#70] -Arguments: [cs_sold_date_sk#63], [d_date_sk#69], Inner, BuildRight - -(55) CometProject -Input [11]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_date_sk#69, d_year#70] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70], [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] - -(56) CometExchange -Input [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] -Arguments: hashpartitioning(cs_order_number#60, cs_item_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(57) CometSort -Input [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70], [cs_order_number#60 ASC NULLS FIRST, cs_item_sk#59 ASC NULLS FIRST] - -(58) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] - -(59) CometSort -Input [4]: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] -Arguments: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74], [cr_order_number#72 ASC NULLS FIRST, cr_item_sk#71 ASC NULLS FIRST] - -(60) CometSortMergeJoin -Left output [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] -Right output [4]: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] -Arguments: [cs_order_number#60, cs_item_sk#59], [cr_order_number#72, cr_item_sk#71], LeftOuter - -(61) CometProject -Input [13]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70, cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] -Arguments: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76], [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, (cs_quantity#61 - coalesce(cr_return_quantity#73, 0)) AS sales_cnt#75, (cs_ext_sales_price#62 - coalesce(cr_return_amount#74, 0.00)) AS sales_amt#76] - -(62) ReusedExchange [Reuses operator id: 56] -Output [9]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85] - -(63) CometSort -Input [9]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85] -Arguments: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85], [ss_ticket_number#78 ASC NULLS FIRST, ss_item_sk#77 ASC NULLS FIRST] - -(64) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] - -(65) CometSort -Input [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] -Arguments: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89], [sr_ticket_number#87 ASC NULLS FIRST, sr_item_sk#86 ASC NULLS FIRST] +Aggregate Attributes [2]: [sum(sales_cnt#19)#61, sum(UnscaledValue(sales_amt#20))#62] +Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#61 AS sales_cnt#63, MakeDecimal(sum(UnscaledValue(sales_amt#20))#62,18,2) AS sales_amt#64] + +(65) Filter [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Condition : isnotnull(sales_cnt#63) + +(66) CometColumnarExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(67) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] + +(68) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#69)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(69) CometFilter +Input [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Condition : isnotnull(cs_item_sk#65) + +(70) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(71) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Right output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65], [i_item_sk#70], Inner, BuildRight + +(72) CometProject +Input [10]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(73) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#75, d_year#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(74) CometFilter +Input [2]: [d_date_sk#75, d_year#76] +Condition : ((isnotnull(d_year#76) AND (d_year#76 = 2001)) AND isnotnull(d_date_sk#75)) + +(75) CometBroadcastExchange +Input [2]: [d_date_sk#75, d_year#76] +Arguments: [d_date_sk#75, d_year#76] + +(76) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Right output [2]: [d_date_sk#75, d_year#76] +Arguments: [cs_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight + +(77) CometProject +Input [11]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_date_sk#75, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] + +(78) CometExchange +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: hashpartitioning(cs_order_number#66, cs_item_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(79) CometSort +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_order_number#66 ASC NULLS FIRST, cs_item_sk#65 ASC NULLS FIRST] + +(80) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] + +(81) CometSort +Input [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80], [cr_order_number#78 ASC NULLS FIRST, cr_item_sk#77 ASC NULLS FIRST] + +(82) CometSortMergeJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Right output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cs_order_number#66, cs_item_sk#65], [cr_order_number#78, cr_item_sk#77], LeftOuter + +(83) CometProject +Input [13]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76, cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82], [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, (cs_quantity#67 - coalesce(cr_return_quantity#79, 0)) AS sales_cnt#81, (cs_ext_sales_price#68 - coalesce(cr_return_amount#80, 0.00)) AS sales_amt#82] + +(84) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(85) CometFilter +Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Condition : isnotnull(ss_item_sk#83) + +(86) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(87) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Right output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83], [i_item_sk#88], Inner, BuildRight + +(88) CometProject +Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(89) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#93, d_year#94] + +(90) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ss_sold_date_sk#87], [d_date_sk#93], Inner, BuildRight + +(91) CometProject +Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] + +(92) CometExchange +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(93) CometSort +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] + +(94) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] + +(95) CometSort +Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98], [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST] + +(96) CometSortMergeJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Right output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#96, sr_item_sk#95], LeftOuter + +(97) CometProject +Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100], [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#99, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#100] + +(98) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#105)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(99) CometFilter +Input [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Condition : isnotnull(ws_item_sk#101) + +(100) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(101) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101], [i_item_sk#106], Inner, BuildRight + +(102) CometProject +Input [10]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(103) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#111, d_year#112] + +(104) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Right output [2]: [d_date_sk#111, d_year#112] +Arguments: [ws_sold_date_sk#105], [d_date_sk#111], Inner, BuildRight + +(105) CometProject +Input [11]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] + +(106) CometExchange +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: hashpartitioning(ws_order_number#102, ws_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(107) CometSort +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#102 ASC NULLS FIRST, ws_item_sk#101 ASC NULLS FIRST] + +(108) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] + +(109) CometSort +Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] + +(110) CometSortMergeJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [ws_order_number#102, ws_item_sk#101], [wr_order_number#114, wr_item_sk#113], LeftOuter + +(111) CometProject +Input [13]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#103 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#117, (ws_ext_sales_price#104 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#118] + +(112) CometUnion +Child 0 Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Child 1 Input [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100] +Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118] + +(113) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] -(66) CometSortMergeJoin -Left output [9]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85] -Right output [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] -Arguments: [ss_ticket_number#78, ss_item_sk#77], [sr_ticket_number#87, sr_item_sk#86], LeftOuter +(114) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(67) CometProject -Input [13]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85, sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] -Arguments: [d_year#85, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, sales_cnt#90, sales_amt#91], [d_year#85, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, (ss_quantity#79 - coalesce(sr_return_quantity#88, 0)) AS sales_cnt#90, (ss_ext_sales_price#80 - coalesce(sr_return_amt#89, 0.00)) AS sales_amt#91] +(115) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] -(68) ReusedExchange [Reuses operator id: 56] -Output [9]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100] +(116) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [partial_sum(sales_cnt#81), partial_sum(UnscaledValue(sales_amt#82))] -(69) CometSort -Input [9]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100] -Arguments: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100], [ws_order_number#93 ASC NULLS FIRST, ws_item_sk#92 ASC NULLS FIRST] +(117) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(70) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] +(118) CometColumnarToRow [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] -(71) CometSort -Input [4]: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] -Arguments: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104], [wr_order_number#102 ASC NULLS FIRST, wr_item_sk#101 ASC NULLS FIRST] +(119) HashAggregate [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [sum(sales_cnt#81), sum(UnscaledValue(sales_amt#82))] +Aggregate Attributes [2]: [sum(sales_cnt#81)#61, sum(UnscaledValue(sales_amt#82))#62] +Results [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum(sales_cnt#81)#61 AS sales_cnt#121, MakeDecimal(sum(UnscaledValue(sales_amt#82))#62,18,2) AS sales_amt#122] -(72) CometSortMergeJoin -Left output [9]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100] -Right output [4]: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] -Arguments: [ws_order_number#93, ws_item_sk#92], [wr_order_number#102, wr_item_sk#101], LeftOuter +(120) Filter [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Condition : isnotnull(sales_cnt#121) -(73) CometProject -Input [13]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100, wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] -Arguments: [d_year#100, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#105, sales_amt#106], [d_year#100, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, (ws_quantity#94 - coalesce(wr_return_quantity#103, 0)) AS sales_cnt#105, (ws_ext_sales_price#95 - coalesce(wr_return_amt#104, 0.00)) AS sales_amt#106] +(121) CometColumnarExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: hashpartitioning(i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] -(74) CometUnion -Child 0 Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Child 1 Input [7]: [d_year#85, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, sales_cnt#90, sales_amt#91] -Child 2 Input [7]: [d_year#100, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#105, sales_amt#106] +(122) CometSort +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122], [i_brand_id#71 ASC NULLS FIRST, i_class_id#72 ASC NULLS FIRST, i_category_id#73 ASC NULLS FIRST, i_manufact_id#74 ASC NULLS FIRST] -(75) CometHashAggregate -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Keys [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Functions: [] +(123) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Right output [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], Inner, ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#121 as decimal(17,2))) < 0.90000000000000000000) -(76) CometExchange -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Arguments: hashpartitioning(d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(124) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], [d_year#76 AS prev_year#123, d_year#13 AS year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#121 AS prev_yr_cnt#125, sales_cnt#63 AS curr_yr_cnt#126, (sales_cnt#63 - sales_cnt#121) AS sales_cnt_diff#127, (sales_amt#64 - sales_amt#122) AS sales_amt_diff#128] -(77) CometHashAggregate -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Keys [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Functions: [] +(125) CometTakeOrderedAndProject +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#127 ASC NULLS FIRST], output=[prev_year#123,year#124,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#125,curr_yr_cnt#126,sales_cnt_diff#127,sales_amt_diff#128]), [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], 100, 0, [sales_cnt_diff#127 ASC NULLS FIRST], [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] -(78) ColumnarToRow [codegen id : 4] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] - -(79) HashAggregate [codegen id : 4] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Keys [5]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Functions [2]: [partial_sum(sales_cnt#75), partial_sum(UnscaledValue(sales_amt#76))] -Aggregate Attributes [2]: [sum#107, sum#108] -Results [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum#109, sum#110] - -(80) Exchange -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum#109, sum#110] -Arguments: hashpartitioning(d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(81) HashAggregate [codegen id : 5] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum#109, sum#110] -Keys [5]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Functions [2]: [sum(sales_cnt#75), sum(UnscaledValue(sales_amt#76))] -Aggregate Attributes [2]: [sum(sales_cnt#75)#55, sum(UnscaledValue(sales_amt#76))#56] -Results [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum(sales_cnt#75)#55 AS sales_cnt#111, MakeDecimal(sum(UnscaledValue(sales_amt#76))#56,18,2) AS sales_amt#112] - -(82) Filter [codegen id : 5] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] -Condition : isnotnull(sales_cnt#111) - -(83) Exchange -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] -Arguments: hashpartitioning(i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(84) Sort [codegen id : 6] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] -Arguments: [i_brand_id#65 ASC NULLS FIRST, i_class_id#66 ASC NULLS FIRST, i_category_id#67 ASC NULLS FIRST, i_manufact_id#68 ASC NULLS FIRST], false, 0 - -(85) SortMergeJoin [codegen id : 7] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Join type: Inner -Join condition: ((cast(sales_cnt#57 as decimal(17,2)) / cast(sales_cnt#111 as decimal(17,2))) < 0.90000000000000000000) - -(86) Project [codegen id : 7] -Output [10]: [d_year#70 AS prev_year#113, d_year#13 AS year#114, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#111 AS prev_yr_cnt#115, sales_cnt#57 AS curr_yr_cnt#116, (sales_cnt#57 - sales_cnt#111) AS sales_cnt_diff#117, (sales_amt#58 - sales_amt#112) AS sales_amt_diff#118] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58, d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] - -(87) TakeOrderedAndProject -Input [10]: [prev_year#113, year#114, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#115, curr_yr_cnt#116, sales_cnt_diff#117, sales_amt_diff#118] -Arguments: 100, [sales_cnt_diff#117 ASC NULLS FIRST], [prev_year#113, year#114, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#115, curr_yr_cnt#116, sales_cnt_diff#117, sales_amt_diff#118] +(126) CometColumnarToRow [codegen id : 3] +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/extended.txt new file mode 100644 index 0000000000..dc2ec3af0b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/extended.txt @@ -0,0 +1,160 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#1, i_brand_id#2, i_class_id#3, i_category_id#4, i_manufact_id#5, sum(sales_cnt#6)#7 AS sales_cnt#8, MakeDecimal(sum(UnscaledValue(sales_amt#9))#10,18,2) AS sales_amt#11)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_returns + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_returns + +- CometSort + +- CometColumnarExchange + +- Filter + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#12, i_brand_id#13, i_class_id#14, i_category_id#15, i_manufact_id#16, sum(sales_cnt#17)#7 AS sales_cnt#18, MakeDecimal(sum(UnscaledValue(sales_amt#19))#10,18,2) AS sales_amt#20)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_returns + +Comet accelerated 151 out of 155 eligible operators (97%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/simplified.txt index 2d5d184d6e..557bee8d99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/simplified.txt @@ -1,102 +1,132 @@ -TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_amt_diff] - WholeStageCodegen (7) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (3) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (2) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] +WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + WholeStageCodegen (1) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (1) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #4 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #7 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 - InputAdapter - WholeStageCodegen (6) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #8 - WholeStageCodegen (5) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + WholeStageCodegen (2) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #9 - WholeStageCodegen (4) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #10 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #11 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometBroadcastExchange [d_date_sk,d_year] #12 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/explain.txt index bbead60635..780930f39f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/explain.txt @@ -1,181 +1,202 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometUnion (29) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (35) ++- * HashAggregate (34) + +- * CometColumnarToRow (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometUnion (30) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - :- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : +- ReusedExchange (15) - : +- ReusedExchange (18) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (21) - : +- ReusedExchange (23) - +- ReusedExchange (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + :- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (14) + : : +- ReusedExchange (16) + : +- ReusedExchange (19) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometFilter (23) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (22) + : +- ReusedExchange (24) + +- ReusedExchange (27) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#5, i_category#6] Condition : isnotnull(i_item_sk#5) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] +Arguments: [i_item_sk#5, i_category#7], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#6, 50)) AS i_category#7] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#5, i_category#7] +Arguments: [i_item_sk#5, i_category#7] + +(7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [2]: [i_item_sk#5, i_category#6] +Right output [2]: [i_item_sk#5, i_category#7] Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight -(7) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(9) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(11) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Right output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [ss_sold_date_sk#4], [d_date_sk#7], Inner, BuildRight - -(12) CometProject -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12], [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Arguments: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] - -(14) CometFilter -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) - -(15) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#17, i_category#18] - -(16) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Right output [2]: [i_item_sk#17, i_category#18] -Arguments: [ws_item_sk#13], [i_item_sk#17], Inner, BuildRight - -(17) CometProject -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] -Arguments: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18], [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] - -(18) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] - -(19) CometBroadcastHashJoin -Left output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Right output [3]: [d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [ws_sold_date_sk#16], [d_date_sk#19], Inner, BuildRight - -(20) CometProject -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24], [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] - -(21) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Arguments: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] - -(22) CometFilter -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) - -(23) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#29, i_category#30] - -(24) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Right output [2]: [i_item_sk#29, i_category#30] -Arguments: [cs_item_sk#26], [i_item_sk#29], Inner, BuildRight - -(25) CometProject -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] -Arguments: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30], [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] - -(26) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] - -(27) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Right output [3]: [d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [cs_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight - -(28) CometProject -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36], [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] - -(29) CometUnion -Child 0 Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Child 1 Input [6]: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24] -Child 2 Input [6]: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36] - -(30) ColumnarToRow [codegen id : 1] -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] - -(31) HashAggregate [codegen id : 1] -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count#37, sum#38] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] - -(32) Exchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(33) HashAggregate [codegen id : 2] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count(1)#41, sum(UnscaledValue(ext_sales_price#12))#42] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count(1)#41 AS sales_cnt#43, MakeDecimal(sum(UnscaledValue(ext_sales_price#12))#42,17,2) AS sales_amt#44] - -(34) TakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] -Arguments: 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#7] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : isnotnull(d_date_sk#8) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8, d_year#9, d_qoy#10] + +(12) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] +Right output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [ss_sold_date_sk#4], [d_date_sk#8], Inner, BuildRight + +(13) CometProject +Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7, d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13], [store AS channel#11, ss_store_sk#2 AS col_name#12, d_year#9, d_qoy#10, i_category#7, ss_ext_sales_price#3 AS ext_sales_price#13] + +(14) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(15) CometFilter +Input [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Condition : (isnull(ws_ship_customer_sk#15) AND isnotnull(ws_item_sk#14)) + +(16) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#18, i_category#19] + +(17) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Right output [2]: [i_item_sk#18, i_category#19] +Arguments: [ws_item_sk#14], [i_item_sk#18], Inner, BuildRight + +(18) CometProject +Input [6]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_item_sk#18, i_category#19] +Arguments: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19], [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] + +(19) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#20, d_year#21, d_qoy#22] + +(20) CometBroadcastHashJoin +Left output [4]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] +Right output [3]: [d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight + +(21) CometProject +Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] + +(22) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] +PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(23) CometFilter +Input [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Condition : (isnull(cs_ship_addr_sk#26) AND isnotnull(cs_item_sk#27)) + +(24) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#30, i_category#31] + +(25) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Right output [2]: [i_item_sk#30, i_category#31] +Arguments: [cs_item_sk#27], [i_item_sk#30], Inner, BuildRight + +(26) CometProject +Input [6]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29, i_item_sk#30, i_category#31] +Arguments: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31], [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] + +(27) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#32, d_year#33, d_qoy#34] + +(28) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] +Right output [3]: [d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [cs_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight + +(29) CometProject +Input [7]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31, d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37], [catalog AS channel#35, cs_ship_addr_sk#26 AS col_name#36, d_year#33, d_qoy#34, i_category#31, cs_ext_sales_price#28 AS ext_sales_price#37] + +(30) CometUnion +Child 0 Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Child 1 Input [6]: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25] +Child 2 Input [6]: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37] + +(31) CometHashAggregate +Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#13))] + +(32) CometExchange +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Arguments: hashpartitioning(channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(33) CometColumnarToRow [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] + +(34) HashAggregate [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#13))] +Aggregate Attributes [2]: [count(1)#40, sum(UnscaledValue(ext_sales_price#13))#41] +Results [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count(1)#40 AS sales_cnt#42, MakeDecimal(sum(UnscaledValue(ext_sales_price#13))#41,17,2) AS sales_amt#43] + +(35) TakeOrderedAndProject +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#42, sales_amt#43] +Arguments: 100, [channel#11 ASC NULLS FIRST, col_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, i_category#7 ASC NULLS FIRST], [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#42, sales_amt#43] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/extended.txt new file mode 100644 index 0000000000..bc57f34dc0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/extended.txt @@ -0,0 +1,47 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(channel#1, col_name#2, d_year#3, d_qoy#4, i_category#5, count(1)#6 AS sales_cnt#7, MakeDecimal(sum(UnscaledValue(ext_sales_price#8))#9,17,2) AS sales_amt#10)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 42 out of 44 eligible operators (95%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/simplified.txt index b3603c6017..63df5f2829 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/simplified.txt @@ -1,38 +1,37 @@ TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum] - InputAdapter - Exchange [channel,col_name,d_year,d_qoy,i_category] #1 - WholeStageCodegen (1) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum] - ColumnarToRow - InputAdapter - CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category] #2 - CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] - CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] - CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 + CometColumnarToRow + InputAdapter + CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 + CometHashAggregate [ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,count,sum] + CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category] #2 + CometProject [i_category] [i_item_sk,i_category] + CometFilter [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] + CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] + CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/explain.txt index f12c117aba..ff59276044 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/explain.txt @@ -1,69 +1,111 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Expand (48) - +- Union (47) - :- * Project (22) - : +- * BroadcastHashJoin LeftOuter BuildRight (21) - : :- * HashAggregate (17) - : : +- Exchange (16) - : : +- * HashAggregate (15) - : : +- * ColumnarToRow (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- BroadcastExchange (20) - : +- * HashAggregate (19) - : +- ReusedExchange (18) - :- * Project (41) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (40) - : :- BroadcastExchange (31) - : : +- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * ColumnarToRow (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (23) - : : +- ReusedExchange (24) - : +- * HashAggregate (39) - : +- Exchange (38) - : +- * HashAggregate (37) - : +- * ColumnarToRow (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (32) - : +- ReusedExchange (33) - +- * Project (46) - +- * BroadcastHashJoin LeftOuter BuildRight (45) - :- * HashAggregate (43) - : +- ReusedExchange (42) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (87) ++- * HashAggregate (86) + +- * CometColumnarToRow (85) + +- CometColumnarExchange (84) + +- * HashAggregate (83) + +- * Expand (82) + +- Union (81) + :- * Project (32) + : +- * BroadcastHashJoin LeftOuter BuildRight (31) + : :- * HashAggregate (17) + : : +- * CometColumnarToRow (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- BroadcastExchange (30) + : +- * HashAggregate (29) + : +- * CometColumnarToRow (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + :- * Project (51) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (50) + : :- BroadcastExchange (41) + : : +- * HashAggregate (40) + : : +- * CometColumnarToRow (39) + : : +- CometExchange (38) + : : +- CometHashAggregate (37) + : : +- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (33) + : : +- ReusedExchange (34) + : +- * HashAggregate (49) + : +- * CometColumnarToRow (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometNativeScan parquet spark_catalog.default.catalog_returns (42) + : +- ReusedExchange (43) + +- * Project (80) + +- * BroadcastHashJoin LeftOuter BuildRight (79) + :- * HashAggregate (65) + : +- * CometColumnarToRow (64) + : +- CometExchange (63) + : +- CometHashAggregate (62) + : +- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometFilter (53) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (52) + : : +- ReusedExchange (54) + : +- CometBroadcastExchange (59) + : +- CometFilter (58) + : +- CometNativeScan parquet spark_catalog.default.web_page (57) + +- BroadcastExchange (78) + +- * HashAggregate (77) + +- * CometColumnarToRow (76) + +- CometExchange (75) + +- CometHashAggregate (74) + +- CometProject (73) + +- CometBroadcastHashJoin (72) + :- CometProject (70) + : +- CometBroadcastHashJoin (69) + : :- CometFilter (67) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (66) + : +- ReusedExchange (68) + +- ReusedExchange (71) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_date#6] @@ -86,9 +128,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [1]: [s_store_sk#7] @@ -107,185 +152,346 @@ Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -(14) ColumnarToRow [codegen id : 1] -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] - -(15) HashAggregate [codegen id : 1] +(14) CometHashAggregate Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] -(16) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] -(17) HashAggregate [codegen id : 4] -Input [3]: [s_store_sk#7, sum#10, sum#11] +(17) HashAggregate [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] - -(18) ReusedExchange [Reuses operator id: 16] -Output [3]: [s_store_sk#16, sum#17, sum#18] - -(19) HashAggregate [codegen id : 3] -Input [3]: [s_store_sk#16, sum#17, sum#18] -Keys [1]: [s_store_sk#16] -Functions [2]: [sum(UnscaledValue(sr_return_amt#19)), sum(UnscaledValue(sr_net_loss#20))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#19))#21, sum(UnscaledValue(sr_net_loss#20))#22] -Results [3]: [s_store_sk#16, MakeDecimal(sum(UnscaledValue(sr_return_amt#19))#21,17,2) AS returns#23, MakeDecimal(sum(UnscaledValue(sr_net_loss#20))#22,17,2) AS profit_loss#24] - -(20) BroadcastExchange -Input [3]: [s_store_sk#16, returns#23, profit_loss#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(21) BroadcastHashJoin [codegen id : 4] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#10, sum(UnscaledValue(ss_net_profit#3))#11] +Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS sales#12, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#11,17,2) AS profit#13] + +(18) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#17)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(19) CometFilter +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Condition : isnotnull(sr_store_sk#14) + +(20) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] + +(21) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight + +(22) CometProject +Input [5]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17, d_date_sk#18] +Arguments: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16], [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] + +(23) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#19] + +(24) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] +Right output [1]: [s_store_sk#19] +Arguments: [sr_store_sk#14], [s_store_sk#19], Inner, BuildRight + +(25) CometProject +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Arguments: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19], [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] + +(26) CometHashAggregate +Input [3]: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Keys [1]: [s_store_sk#19] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#15)), partial_sum(UnscaledValue(sr_net_loss#16))] + +(27) CometExchange +Input [3]: [s_store_sk#19, sum#20, sum#21] +Arguments: hashpartitioning(s_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] + +(29) HashAggregate [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Keys [1]: [s_store_sk#19] +Functions [2]: [sum(UnscaledValue(sr_return_amt#15)), sum(UnscaledValue(sr_net_loss#16))] +Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#15))#22, sum(UnscaledValue(sr_net_loss#16))#23] +Results [3]: [s_store_sk#19, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#22,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(sr_net_loss#16))#23,17,2) AS profit_loss#25] + +(30) BroadcastExchange +Input [3]: [s_store_sk#19, returns#24, profit_loss#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(31) BroadcastHashJoin [codegen id : 2] Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#16] +Right keys [1]: [s_store_sk#19] Join type: LeftOuter Join condition: None -(22) Project [codegen id : 4] -Output [5]: [sales#14, coalesce(returns#23, 0.00) AS returns#25, (profit#15 - coalesce(profit_loss#24, 0.00)) AS profit#26, store channel AS channel#27, s_store_sk#7 AS id#28] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#16, returns#23, profit_loss#24] - -(23) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32] -Arguments: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32] - -(24) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] - -(25) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32] -Right output [1]: [d_date_sk#33] -Arguments: [cs_sold_date_sk#32], [d_date_sk#33], Inner, BuildRight - -(26) CometProject -Input [5]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32, d_date_sk#33] -Arguments: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31], [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31] - -(27) ColumnarToRow [codegen id : 5] -Input [3]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31] - -(28) HashAggregate [codegen id : 5] -Input [3]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31] -Keys [1]: [cs_call_center_sk#29] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#30)), partial_sum(UnscaledValue(cs_net_profit#31))] -Aggregate Attributes [2]: [sum#34, sum#35] -Results [3]: [cs_call_center_sk#29, sum#36, sum#37] - -(29) Exchange -Input [3]: [cs_call_center_sk#29, sum#36, sum#37] -Arguments: hashpartitioning(cs_call_center_sk#29, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(30) HashAggregate [codegen id : 6] -Input [3]: [cs_call_center_sk#29, sum#36, sum#37] -Keys [1]: [cs_call_center_sk#29] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#30)), sum(UnscaledValue(cs_net_profit#31))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#30))#38, sum(UnscaledValue(cs_net_profit#31))#39] -Results [3]: [cs_call_center_sk#29, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#30))#38,17,2) AS sales#40, MakeDecimal(sum(UnscaledValue(cs_net_profit#31))#39,17,2) AS profit#41] - -(31) BroadcastExchange -Input [3]: [cs_call_center_sk#29, sales#40, profit#41] -Arguments: IdentityBroadcastMode, [plan_id=4] - -(32) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44] -Arguments: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44] - -(33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#45] - -(34) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44] -Right output [1]: [d_date_sk#45] -Arguments: [cr_returned_date_sk#44], [d_date_sk#45], Inner, BuildRight - -(35) CometProject -Input [4]: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44, d_date_sk#45] -Arguments: [cr_return_amount#42, cr_net_loss#43], [cr_return_amount#42, cr_net_loss#43] - -(36) ColumnarToRow [codegen id : 7] -Input [2]: [cr_return_amount#42, cr_net_loss#43] - -(37) HashAggregate [codegen id : 7] -Input [2]: [cr_return_amount#42, cr_net_loss#43] +(32) Project [codegen id : 2] +Output [5]: [sales#12, coalesce(returns#24, 0.00) AS returns#26, (profit#13 - coalesce(profit_loss#25, 0.00)) AS profit#27, store channel AS channel#28, s_store_sk#7 AS id#29] +Input [6]: [s_store_sk#7, sales#12, profit#13, s_store_sk#19, returns#24, profit_loss#25] + +(33) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +ReadSchema: struct + +(34) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#34] + +(35) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [1]: [d_date_sk#34] +Arguments: [cs_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight + +(36) CometProject +Input [5]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, d_date_sk#34] +Arguments: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32], [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] + +(37) CometHashAggregate +Input [3]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(UnscaledValue(cs_net_profit#32))] + +(38) CometExchange +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Arguments: hashpartitioning(cs_call_center_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometColumnarToRow [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] + +(40) HashAggregate [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(UnscaledValue(cs_net_profit#32))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#31))#37, sum(UnscaledValue(cs_net_profit#32))#38] +Results [3]: [cs_call_center_sk#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#31))#37,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#32))#38,17,2) AS profit#40] + +(41) BroadcastExchange +Input [3]: [cs_call_center_sk#30, sales#39, profit#40] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(42) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] +ReadSchema: struct + +(43) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#44] + +(44) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Right output [1]: [d_date_sk#44] +Arguments: [cr_returned_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(45) CometProject +Input [4]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43, d_date_sk#44] +Arguments: [cr_return_amount#41, cr_net_loss#42], [cr_return_amount#41, cr_net_loss#42] + +(46) CometHashAggregate +Input [2]: [cr_return_amount#41, cr_net_loss#42] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#42)), partial_sum(UnscaledValue(cr_net_loss#43))] -Aggregate Attributes [2]: [sum#46, sum#47] -Results [2]: [sum#48, sum#49] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#41)), partial_sum(UnscaledValue(cr_net_loss#42))] + +(47) CometExchange +Input [2]: [sum#45, sum#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(38) Exchange -Input [2]: [sum#48, sum#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(48) CometColumnarToRow +Input [2]: [sum#45, sum#46] -(39) HashAggregate -Input [2]: [sum#48, sum#49] +(49) HashAggregate +Input [2]: [sum#45, sum#46] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#42)), sum(UnscaledValue(cr_net_loss#43))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#42))#50, sum(UnscaledValue(cr_net_loss#43))#51] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#42))#50,17,2) AS returns#52, MakeDecimal(sum(UnscaledValue(cr_net_loss#43))#51,17,2) AS profit_loss#53] +Functions [2]: [sum(UnscaledValue(cr_return_amount#41)), sum(UnscaledValue(cr_net_loss#42))] +Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#41))#47, sum(UnscaledValue(cr_net_loss#42))#48] +Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#41))#47,17,2) AS returns#49, MakeDecimal(sum(UnscaledValue(cr_net_loss#42))#48,17,2) AS profit_loss#50] -(40) BroadcastNestedLoopJoin [codegen id : 8] +(50) BroadcastNestedLoopJoin [codegen id : 4] Join type: Inner Join condition: None -(41) Project [codegen id : 8] -Output [5]: [sales#40, returns#52, (profit#41 - profit_loss#53) AS profit#54, catalog channel AS channel#55, cs_call_center_sk#29 AS id#56] -Input [5]: [cs_call_center_sk#29, sales#40, profit#41, returns#52, profit_loss#53] +(51) Project [codegen id : 4] +Output [5]: [sales#39, returns#49, (profit#40 - profit_loss#50) AS profit#51, catalog channel AS channel#52, cs_call_center_sk#30 AS id#53] +Input [5]: [cs_call_center_sk#30, sales#39, profit#40, returns#49, profit_loss#50] + +(52) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#57)] +PushedFilters: [IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(53) CometFilter +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Condition : isnotnull(ws_web_page_sk#54) + +(54) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(55) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [ws_sold_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(56) CometProject +Input [5]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57, d_date_sk#58] +Arguments: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56], [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] + +(57) CometNativeScan parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(58) CometFilter +Input [1]: [wp_web_page_sk#59] +Condition : isnotnull(wp_web_page_sk#59) + +(59) CometBroadcastExchange +Input [1]: [wp_web_page_sk#59] +Arguments: [wp_web_page_sk#59] + +(60) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [ws_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(61) CometProject +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Arguments: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59], [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] + +(62) CometHashAggregate +Input [3]: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#55)), partial_sum(UnscaledValue(ws_net_profit#56))] + +(63) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(64) CometColumnarToRow [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] + +(65) HashAggregate [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#55)), sum(UnscaledValue(ws_net_profit#56))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#55))#62, sum(UnscaledValue(ws_net_profit#56))#63] +Results [3]: [wp_web_page_sk#59, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#55))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#56))#63,17,2) AS profit#65] + +(66) CometNativeScan parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#69)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct + +(67) CometFilter +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Condition : isnotnull(wr_web_page_sk#66) + +(68) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#70] + +(69) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Right output [1]: [d_date_sk#70] +Arguments: [wr_returned_date_sk#69], [d_date_sk#70], Inner, BuildRight + +(70) CometProject +Input [5]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69, d_date_sk#70] +Arguments: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68], [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] + +(71) ReusedExchange [Reuses operator id: 59] +Output [1]: [wp_web_page_sk#71] + +(72) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] +Right output [1]: [wp_web_page_sk#71] +Arguments: [wr_web_page_sk#66], [wp_web_page_sk#71], Inner, BuildRight + +(73) CometProject +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Arguments: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71], [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] + +(74) CometHashAggregate +Input [3]: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#67)), partial_sum(UnscaledValue(wr_net_loss#68))] + +(75) CometExchange +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Arguments: hashpartitioning(wp_web_page_sk#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(76) CometColumnarToRow [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] + +(77) HashAggregate [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [sum(UnscaledValue(wr_return_amt#67)), sum(UnscaledValue(wr_net_loss#68))] +Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#67))#74, sum(UnscaledValue(wr_net_loss#68))#75] +Results [3]: [wp_web_page_sk#71, MakeDecimal(sum(UnscaledValue(wr_return_amt#67))#74,17,2) AS returns#76, MakeDecimal(sum(UnscaledValue(wr_net_loss#68))#75,17,2) AS profit_loss#77] + +(78) BroadcastExchange +Input [3]: [wp_web_page_sk#71, returns#76, profit_loss#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(79) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [wp_web_page_sk#59] +Right keys [1]: [wp_web_page_sk#71] +Join type: LeftOuter +Join condition: None -(42) ReusedExchange [Reuses operator id: 16] -Output [3]: [wp_web_page_sk#57, sum#58, sum#59] +(80) Project [codegen id : 6] +Output [5]: [sales#64, coalesce(returns#76, 0.00) AS returns#78, (profit#65 - coalesce(profit_loss#77, 0.00)) AS profit#79, web channel AS channel#80, wp_web_page_sk#59 AS id#81] +Input [6]: [wp_web_page_sk#59, sales#64, profit#65, wp_web_page_sk#71, returns#76, profit_loss#77] -(43) HashAggregate [codegen id : 12] -Input [3]: [wp_web_page_sk#57, sum#58, sum#59] -Keys [1]: [wp_web_page_sk#57] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#60)), sum(UnscaledValue(ws_net_profit#61))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#60))#62, sum(UnscaledValue(ws_net_profit#61))#63] -Results [3]: [wp_web_page_sk#57, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#60))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#61))#63,17,2) AS profit#65] +(81) Union -(44) ReusedExchange [Reuses operator id: 20] -Output [3]: [wp_web_page_sk#66, returns#67, profit_loss#68] +(82) Expand [codegen id : 7] +Input [5]: [sales#12, returns#26, profit#27, channel#28, id#29] +Arguments: [[sales#12, returns#26, profit#27, channel#28, id#29, 0], [sales#12, returns#26, profit#27, channel#28, null, 1], [sales#12, returns#26, profit#27, null, null, 3]], [sales#12, returns#26, profit#27, channel#82, id#83, spark_grouping_id#84] -(45) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [wp_web_page_sk#57] -Right keys [1]: [wp_web_page_sk#66] -Join type: LeftOuter -Join condition: None +(83) HashAggregate [codegen id : 7] +Input [6]: [sales#12, returns#26, profit#27, channel#82, id#83, spark_grouping_id#84] +Keys [3]: [channel#82, id#83, spark_grouping_id#84] +Functions [3]: [partial_sum(sales#12), partial_sum(returns#26), partial_sum(profit#27)] +Aggregate Attributes [6]: [sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Results [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] + +(84) CometColumnarExchange +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(channel#82, id#83, spark_grouping_id#84, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(85) CometColumnarToRow [codegen id : 8] +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] + +(86) HashAggregate [codegen id : 8] +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Keys [3]: [channel#82, id#83, spark_grouping_id#84] +Functions [3]: [sum(sales#12), sum(returns#26), sum(profit#27)] +Aggregate Attributes [3]: [sum(sales#12)#97, sum(returns#26)#98, sum(profit#27)#99] +Results [5]: [channel#82, id#83, sum(sales#12)#97 AS sales#100, sum(returns#26)#98 AS returns#101, sum(profit#27)#99 AS profit#102] -(46) Project [codegen id : 12] -Output [5]: [sales#64, coalesce(returns#67, 0.00) AS returns#69, (profit#65 - coalesce(profit_loss#68, 0.00)) AS profit#70, web channel AS channel#71, wp_web_page_sk#57 AS id#72] -Input [6]: [wp_web_page_sk#57, sales#64, profit#65, wp_web_page_sk#66, returns#67, profit_loss#68] - -(47) Union - -(48) Expand [codegen id : 13] -Input [5]: [sales#14, returns#25, profit#26, channel#27, id#28] -Arguments: [[sales#14, returns#25, profit#26, channel#27, id#28, 0], [sales#14, returns#25, profit#26, channel#27, null, 1], [sales#14, returns#25, profit#26, null, null, 3]], [sales#14, returns#25, profit#26, channel#73, id#74, spark_grouping_id#75] - -(49) HashAggregate [codegen id : 13] -Input [6]: [sales#14, returns#25, profit#26, channel#73, id#74, spark_grouping_id#75] -Keys [3]: [channel#73, id#74, spark_grouping_id#75] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#25), partial_sum(profit#26)] -Aggregate Attributes [6]: [sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] -Results [9]: [channel#73, id#74, spark_grouping_id#75, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(50) Exchange -Input [9]: [channel#73, id#74, spark_grouping_id#75, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Arguments: hashpartitioning(channel#73, id#74, spark_grouping_id#75, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(51) HashAggregate [codegen id : 14] -Input [9]: [channel#73, id#74, spark_grouping_id#75, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [3]: [channel#73, id#74, spark_grouping_id#75] -Functions [3]: [sum(sales#14), sum(returns#25), sum(profit#26)] -Aggregate Attributes [3]: [sum(sales#14)#88, sum(returns#25)#89, sum(profit#26)#90] -Results [5]: [channel#73, id#74, sum(sales#14)#88 AS sales#91, sum(returns#25)#89 AS returns#92, sum(profit#26)#90 AS profit#93] - -(52) TakeOrderedAndProject -Input [5]: [channel#73, id#74, sales#91, returns#92, profit#93] -Arguments: 100, [channel#73 ASC NULLS FIRST, id#74 ASC NULLS FIRST], [channel#73, id#74, sales#91, returns#92, profit#93] +(87) TakeOrderedAndProject +Input [5]: [channel#82, id#83, sales#100, returns#101, profit#102] +Arguments: 100, [channel#82 ASC NULLS FIRST, id#83 ASC NULLS FIRST], [channel#82, id#83, sales#100, returns#101, profit#102] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/extended.txt new file mode 100644 index 0000000000..b56e80b0a7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/extended.txt @@ -0,0 +1,108 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Union + :- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#1, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#3,17,2) AS sales#4, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#6,17,2) AS profit#7)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#8, MakeDecimal(sum(UnscaledValue(sr_return_amt#9))#10,17,2) AS returns#11, MakeDecimal(sum(UnscaledValue(sr_net_loss#12))#13,17,2) AS profit_loss#14)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- Project + : +- BroadcastNestedLoopJoin + : :- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_call_center_sk#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#16))#17,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(cs_net_profit#19))#20,17,2) AS profit#21)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(cr_return_amount#22))#23,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(cr_net_loss#25))#26,17,2) AS profit_loss#27)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- BroadcastHashJoin + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#28, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#29))#30,17,2) AS sales#31, MakeDecimal(sum(UnscaledValue(ws_net_profit#32))#33,17,2) AS profit#34)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#35, MakeDecimal(sum(UnscaledValue(wr_return_amt#36))#37,17,2) AS returns#38, MakeDecimal(sum(UnscaledValue(wr_net_loss#39))#40,17,2) AS profit_loss#41)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 79 out of 99 eligible operators (79%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/simplified.txt index 65c1a5d6b9..17bf9853f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/simplified.txt @@ -1,75 +1,106 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (14) + WholeStageCodegen (8) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (13) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (4) - Project [sales,returns,profit,profit_loss,s_store_sk] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (7) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (2) + Project [sales,returns,profit,profit_loss,s_store_sk] + BroadcastHashJoin [s_store_sk,s_store_sk] + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_sk] #2 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #4 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] InputAdapter - Exchange [s_store_sk] #2 + BroadcastExchange #5 WholeStageCodegen (1) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #4 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - ReusedExchange [s_store_sk,sum,sum] #2 - WholeStageCodegen (8) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #7 - WholeStageCodegen (5) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + CometExchange [s_store_sk] #6 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #4 + WholeStageCodegen (4) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin InputAdapter - Exchange #8 - WholeStageCodegen (7) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - ColumnarToRow + BroadcastExchange #7 + WholeStageCodegen (3) + HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] + CometColumnarToRow InputAdapter - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (12) - Project [sales,returns,profit,profit_loss,wp_web_page_sk] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometExchange [cs_call_center_sk] #8 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange #9 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + WholeStageCodegen (6) + Project [sales,returns,profit,profit_loss,wp_web_page_sk] + BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #10 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] InputAdapter - ReusedExchange [wp_web_page_sk,sum,sum] #2 - InputAdapter - ReusedExchange [wp_web_page_sk,returns,profit_loss] #5 + BroadcastExchange #12 + WholeStageCodegen (5) + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #13 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/explain.txt index 0c93eeff8b..be72f27595 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/explain.txt @@ -1,14 +1,14 @@ == Physical Plan == -TakeOrderedAndProject (49) -+- * Project (48) - +- * SortMergeJoin Inner (47) - :- * Project (28) - : +- * SortMergeJoin Inner (27) +TakeOrderedAndProject (69) ++- * Project (68) + +- * SortMergeJoin Inner (67) + :- * Project (45) + : +- * SortMergeJoin Inner (44) : :- * Sort (22) : : +- * HashAggregate (21) - : : +- Exchange (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) + : : +- * CometColumnarToRow (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometProject (12) @@ -17,42 +17,66 @@ TakeOrderedAndProject (49) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) : : +- CometBroadcastExchange (15) : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- * Sort (26) - : +- * Filter (25) - : +- * HashAggregate (24) - : +- ReusedExchange (23) - +- * Sort (46) - +- * Filter (45) - +- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (37) - : +- CometFilter (36) - : +- CometSortMergeJoin (35) - : :- CometSort (32) - : : +- CometExchange (31) - : : +- CometFilter (30) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : +- CometSort (34) - : +- ReusedExchange (33) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : +- * Sort (43) + : +- * Filter (42) + : +- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (34) + : : +- CometFilter (33) + : : +- CometSortMergeJoin (32) + : : :- CometSort (26) + : : : +- CometExchange (25) + : : : +- CometFilter (24) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (23) + : : +- CometSort (31) + : : +- CometExchange (30) + : : +- CometProject (29) + : : +- CometFilter (28) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (27) + : +- ReusedExchange (35) + +- * Sort (66) + +- * Filter (65) + +- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometFilter (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometFilter (47) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (46) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.catalog_returns (50) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -66,9 +90,12 @@ Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (6) CometFilter Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] @@ -99,9 +126,12 @@ Condition : isnull(sr_ticket_number#9) Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [2]: [d_date_sk#11, d_year#12] @@ -120,147 +150,240 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -(18) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] - -(19) HashAggregate [codegen id : 1] +(18) CometHashAggregate Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -(20) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(20) CometColumnarToRow [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] -(21) HashAggregate [codegen id : 2] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] +(21) HashAggregate [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] - -(22) Sort [codegen id : 2] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 - -(23) ReusedExchange [Reuses operator id: 20] -Output [6]: [d_year#26, ws_item_sk#27, ws_bill_customer_sk#28, sum#29, sum#30, sum#31] - -(24) HashAggregate [codegen id : 4] -Input [6]: [d_year#26, ws_item_sk#27, ws_bill_customer_sk#28, sum#29, sum#30, sum#31] -Keys [3]: [d_year#26, ws_item_sk#27, ws_bill_customer_sk#28] -Functions [3]: [sum(ws_quantity#32), sum(UnscaledValue(ws_wholesale_cost#33)), sum(UnscaledValue(ws_sales_price#34))] -Aggregate Attributes [3]: [sum(ws_quantity#32)#35, sum(UnscaledValue(ws_wholesale_cost#33))#36, sum(UnscaledValue(ws_sales_price#34))#37] -Results [6]: [d_year#26 AS ws_sold_year#38, ws_item_sk#27, ws_bill_customer_sk#28 AS ws_customer_sk#39, sum(ws_quantity#32)#35 AS ws_qty#40, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#33))#36,17,2) AS ws_wc#41, MakeDecimal(sum(UnscaledValue(ws_sales_price#34))#37,17,2) AS ws_sp#42] - -(25) Filter [codegen id : 4] -Input [6]: [ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39, ws_qty#40, ws_wc#41, ws_sp#42] -Condition : (coalesce(ws_qty#40, 0) > 0) - -(26) Sort [codegen id : 4] -Input [6]: [ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39, ws_qty#40, ws_wc#41, ws_sp#42] -Arguments: [ws_sold_year#38 ASC NULLS FIRST, ws_item_sk#27 ASC NULLS FIRST, ws_customer_sk#39 ASC NULLS FIRST], false, 0 - -(27) SortMergeJoin [codegen id : 5] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39] +Aggregate Attributes [3]: [sum(ss_quantity#4)#16, sum(UnscaledValue(ss_wholesale_cost#5))#17, sum(UnscaledValue(ss_sales_price#6))#18] +Results [6]: [d_year#12 AS ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#16 AS ss_qty#20, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#17,17,2) AS ss_wc#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#18,17,2) AS ss_sp#22] + +(22) Sort [codegen id : 1] +Input [6]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 + +(23) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(24) CometFilter +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_item_sk#23) AND isnotnull(ws_bill_customer_sk#24)) + +(25) CometExchange +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: hashpartitioning(ws_order_number#25, ws_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometSort +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_order_number#25 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST] + +(27) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Condition : (isnotnull(wr_order_number#31) AND isnotnull(wr_item_sk#30)) + +(29) CometProject +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_item_sk#30, wr_order_number#31] + +(30) CometExchange +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: hashpartitioning(wr_order_number#31, wr_item_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(31) CometSort +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_order_number#31 ASC NULLS FIRST, wr_item_sk#30 ASC NULLS FIRST] + +(32) CometSortMergeJoin +Left output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [ws_order_number#25, ws_item_sk#23], [wr_order_number#31, wr_item_sk#30], LeftOuter + +(33) CometFilter +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Condition : isnull(wr_order_number#31) + +(34) CometProject +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] + +(35) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#33, d_year#34] + +(36) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [d_date_sk#33, d_year#34] +Arguments: [ws_sold_date_sk#29], [d_date_sk#33], Inner, BuildRight + +(37) CometProject +Input [8]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, d_date_sk#33, d_year#34] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] + +(38) CometHashAggregate +Input [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [partial_sum(ws_quantity#26), partial_sum(UnscaledValue(ws_wholesale_cost#27)), partial_sum(UnscaledValue(ws_sales_price#28))] + +(39) CometExchange +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Arguments: hashpartitioning(d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(40) CometColumnarToRow [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] + +(41) HashAggregate [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +Aggregate Attributes [3]: [sum(ws_quantity#26)#38, sum(UnscaledValue(ws_wholesale_cost#27))#39, sum(UnscaledValue(ws_sales_price#28))#40] +Results [6]: [d_year#34 AS ws_sold_year#41, ws_item_sk#23, ws_bill_customer_sk#24 AS ws_customer_sk#42, sum(ws_quantity#26)#38 AS ws_qty#43, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#27))#39,17,2) AS ws_wc#44, MakeDecimal(sum(UnscaledValue(ws_sales_price#28))#40,17,2) AS ws_sp#45] + +(42) Filter [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Condition : (coalesce(ws_qty#43, 0) > 0) + +(43) Sort [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Arguments: [ws_sold_year#41 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST, ws_customer_sk#42 ASC NULLS FIRST], false, 0 + +(44) SortMergeJoin [codegen id : 3] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42] Join type: Inner Join condition: None -(28) Project [codegen id : 5] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, ws_wc#41, ws_sp#42] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39, ws_qty#40, ws_wc#41, ws_sp#42] - -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] - -(30) CometFilter -Input [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Condition : (isnotnull(cs_item_sk#44) AND isnotnull(cs_bill_customer_sk#43)) - -(31) CometExchange -Input [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Arguments: hashpartitioning(cs_order_number#45, cs_item_sk#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(32) CometSort -Input [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49], [cs_order_number#45 ASC NULLS FIRST, cs_item_sk#44 ASC NULLS FIRST] - -(33) ReusedExchange [Reuses operator id: 8] -Output [2]: [cr_item_sk#50, cr_order_number#51] - -(34) CometSort -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_order_number#51 ASC NULLS FIRST, cr_item_sk#50 ASC NULLS FIRST] - -(35) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Right output [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cs_order_number#45, cs_item_sk#44], [cr_order_number#51, cr_item_sk#50], LeftOuter - -(36) CometFilter -Input [9]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49, cr_item_sk#50, cr_order_number#51] -Condition : isnull(cr_order_number#51) - -(37) CometProject -Input [9]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49, cr_item_sk#50, cr_order_number#51] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49], [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] - -(38) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#52, d_year#53] - -(39) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Right output [2]: [d_date_sk#52, d_year#53] -Arguments: [cs_sold_date_sk#49], [d_date_sk#52], Inner, BuildRight - -(40) CometProject -Input [8]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49, d_date_sk#52, d_year#53] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53], [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53] - -(41) ColumnarToRow [codegen id : 6] -Input [6]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53] - -(42) HashAggregate [codegen id : 6] -Input [6]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53] -Keys [3]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43] -Functions [3]: [partial_sum(cs_quantity#46), partial_sum(UnscaledValue(cs_wholesale_cost#47)), partial_sum(UnscaledValue(cs_sales_price#48))] -Aggregate Attributes [3]: [sum#54, sum#55, sum#56] -Results [6]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, sum#57, sum#58, sum#59] - -(43) Exchange -Input [6]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, sum#57, sum#58, sum#59] -Arguments: hashpartitioning(d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(44) HashAggregate [codegen id : 7] -Input [6]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, sum#57, sum#58, sum#59] -Keys [3]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43] -Functions [3]: [sum(cs_quantity#46), sum(UnscaledValue(cs_wholesale_cost#47)), sum(UnscaledValue(cs_sales_price#48))] -Aggregate Attributes [3]: [sum(cs_quantity#46)#60, sum(UnscaledValue(cs_wholesale_cost#47))#61, sum(UnscaledValue(cs_sales_price#48))#62] -Results [6]: [d_year#53 AS cs_sold_year#63, cs_item_sk#44, cs_bill_customer_sk#43 AS cs_customer_sk#64, sum(cs_quantity#46)#60 AS cs_qty#65, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#47))#61,17,2) AS cs_wc#66, MakeDecimal(sum(UnscaledValue(cs_sales_price#48))#62,17,2) AS cs_sp#67] - -(45) Filter [codegen id : 7] -Input [6]: [cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64, cs_qty#65, cs_wc#66, cs_sp#67] -Condition : (coalesce(cs_qty#65, 0) > 0) - -(46) Sort [codegen id : 7] -Input [6]: [cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64, cs_qty#65, cs_wc#66, cs_sp#67] -Arguments: [cs_sold_year#63 ASC NULLS FIRST, cs_item_sk#44 ASC NULLS FIRST, cs_customer_sk#64 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 8] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64] +(45) Project [codegen id : 3] +Output [9]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45] +Input [12]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] + +(46) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#52)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(47) CometFilter +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Condition : (isnotnull(cs_item_sk#47) AND isnotnull(cs_bill_customer_sk#46)) + +(48) CometExchange +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: hashpartitioning(cs_order_number#48, cs_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(49) CometSort +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_order_number#48 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST] + +(50) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Condition : (isnotnull(cr_order_number#54) AND isnotnull(cr_item_sk#53)) + +(52) CometProject +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_item_sk#53, cr_order_number#54] + +(53) CometExchange +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: hashpartitioning(cr_order_number#54, cr_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(54) CometSort +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_order_number#54 ASC NULLS FIRST, cr_item_sk#53 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cs_order_number#48, cs_item_sk#47], [cr_order_number#54, cr_item_sk#53], LeftOuter + +(56) CometFilter +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Condition : isnull(cr_order_number#54) + +(57) CometProject +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] + +(58) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#56, d_year#57] + +(59) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [d_date_sk#56, d_year#57] +Arguments: [cs_sold_date_sk#52], [d_date_sk#56], Inner, BuildRight + +(60) CometProject +Input [8]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, d_date_sk#56, d_year#57] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] + +(61) CometHashAggregate +Input [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [partial_sum(cs_quantity#49), partial_sum(UnscaledValue(cs_wholesale_cost#50)), partial_sum(UnscaledValue(cs_sales_price#51))] + +(62) CometExchange +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Arguments: hashpartitioning(d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(63) CometColumnarToRow [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] + +(64) HashAggregate [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [sum(cs_quantity#49), sum(UnscaledValue(cs_wholesale_cost#50)), sum(UnscaledValue(cs_sales_price#51))] +Aggregate Attributes [3]: [sum(cs_quantity#49)#61, sum(UnscaledValue(cs_wholesale_cost#50))#62, sum(UnscaledValue(cs_sales_price#51))#63] +Results [6]: [d_year#57 AS cs_sold_year#64, cs_item_sk#47, cs_bill_customer_sk#46 AS cs_customer_sk#65, sum(cs_quantity#49)#61 AS cs_qty#66, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#50))#62,17,2) AS cs_wc#67, MakeDecimal(sum(UnscaledValue(cs_sales_price#51))#63,17,2) AS cs_sp#68] + +(65) Filter [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Condition : (coalesce(cs_qty#66, 0) > 0) + +(66) Sort [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Arguments: [cs_sold_year#64 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST, cs_customer_sk#65 ASC NULLS FIRST], false, 0 + +(67) SortMergeJoin [codegen id : 5] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65] Join type: Inner Join condition: None -(48) Project [codegen id : 8] -Output [12]: [round((cast(ss_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#40 + cs_qty#65), 1) as double)))), 2) AS ratio#68, ss_qty#23 AS store_qty#69, ss_wc#24 AS store_wholesale_cost#70, ss_sp#25 AS store_sales_price#71, (coalesce(ws_qty#40, 0) + coalesce(cs_qty#65, 0)) AS other_chan_qty#72, (coalesce(ws_wc#41, 0.00) + coalesce(cs_wc#66, 0.00)) AS other_chan_wholesale_cost#73, (coalesce(ws_sp#42, 0.00) + coalesce(cs_sp#67, 0.00)) AS other_chan_sales_price#74, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, cs_qty#65] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, ws_wc#41, ws_sp#42, cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64, cs_qty#65, cs_wc#66, cs_sp#67] +(68) Project [codegen id : 5] +Output [12]: [round((cast(ss_qty#20 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)))), 2) AS ratio#69, ss_qty#20 AS store_qty#70, ss_wc#21 AS store_wholesale_cost#71, ss_sp#22 AS store_sales_price#72, (coalesce(ws_qty#43, 0) + coalesce(cs_qty#66, 0)) AS other_chan_qty#73, (coalesce(ws_wc#44, 0.00) + coalesce(cs_wc#67, 0.00)) AS other_chan_wholesale_cost#74, (coalesce(ws_sp#45, 0.00) + coalesce(cs_sp#68, 0.00)) AS other_chan_sales_price#75, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, cs_qty#66] +Input [15]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45, cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] -(49) TakeOrderedAndProject -Input [12]: [ratio#68, store_qty#69, store_wholesale_cost#70, store_sales_price#71, other_chan_qty#72, other_chan_wholesale_cost#73, other_chan_sales_price#74, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, cs_qty#65] -Arguments: 100, [ratio#68 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#72 ASC NULLS FIRST, other_chan_wholesale_cost#73 ASC NULLS FIRST, other_chan_sales_price#74 ASC NULLS FIRST, round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#40 + cs_qty#65), 1) as double)), 2) ASC NULLS FIRST], [ratio#68, store_qty#69, store_wholesale_cost#70, store_sales_price#71, other_chan_qty#72, other_chan_wholesale_cost#73, other_chan_sales_price#74] +(69) TakeOrderedAndProject +Input [12]: [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, cs_qty#66] +Arguments: 100, [ratio#69 ASC NULLS FIRST, ss_qty#20 DESC NULLS LAST, ss_wc#21 DESC NULLS LAST, ss_sp#22 DESC NULLS LAST, other_chan_qty#73 ASC NULLS FIRST, other_chan_wholesale_cost#74 ASC NULLS FIRST, other_chan_sales_price#75 ASC NULLS FIRST, round((cast(ss_qty#20 as double) / cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)), 2) ASC NULLS FIRST], [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/extended.txt new file mode 100644 index 0000000000..52deb08088 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/extended.txt @@ -0,0 +1,75 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- SortMergeJoin + :- Project + : +- SortMergeJoin + : :- Sort + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#1 AS ss_sold_year#2, ss_item_sk#3, ss_customer_sk#4, sum(ss_quantity#5)#6 AS ss_qty#7, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#8))#9,17,2) AS ss_wc#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#11))#12,17,2) AS ss_sp#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- Sort + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#14 AS ws_sold_year#15, ws_item_sk#16, ws_bill_customer_sk#17 AS ws_customer_sk#18, sum(ws_quantity#19)#20 AS ws_qty#21, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#22))#23,17,2) AS ws_wc#24, MakeDecimal(sum(UnscaledValue(ws_sales_price#25))#26,17,2) AS ws_sp#27)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Sort + +- Filter + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#28 AS cs_sold_year#29, cs_item_sk#30, cs_bill_customer_sk#31 AS cs_customer_sk#32, sum(cs_quantity#33)#34 AS cs_qty#35, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#36))#37,17,2) AS cs_wc#38, MakeDecimal(sum(UnscaledValue(cs_sales_price#39))#40,17,2) AS cs_sp#41)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 57 out of 70 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/simplified.txt index 152634bac1..bbadca392a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/simplified.txt @@ -1,65 +1,81 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (8) + WholeStageCodegen (5) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] InputAdapter - WholeStageCodegen (5) + WholeStageCodegen (3) Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] InputAdapter - WholeStageCodegen (2) + WholeStageCodegen (1) Sort [ss_sold_year,ss_item_sk,ss_customer_sk] HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (1) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometExchange [ss_ticket_number,ss_item_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #3 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #4 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometColumnarToRow + InputAdapter + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - WholeStageCodegen (4) + WholeStageCodegen (2) Sort [ws_sold_year,ws_item_sk,ws_customer_sk] Filter [ws_qty] HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - ReusedExchange [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] #1 + CometColumnarToRow + InputAdapter + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 InputAdapter - WholeStageCodegen (7) + WholeStageCodegen (4) Sort [cs_sold_year,cs_item_sk,cs_customer_sk] Filter [cs_qty] HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #5 - WholeStageCodegen (6) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #6 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number] #3 - ReusedExchange [d_date_sk,d_year] #4 + CometColumnarToRow + InputAdapter + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/explain.txt index 7866af00cd..259e4d1217 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) +TakeOrderedAndProject (32) ++- * Project (31) + +- * BroadcastHashJoin Inner BuildRight (30) :- * HashAggregate (24) - : +- Exchange (23) - : +- * HashAggregate (22) - : +- * ColumnarToRow (21) + : +- * CometColumnarToRow (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) : :- CometProject (14) @@ -13,36 +13,44 @@ TakeOrderedAndProject (31) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- BroadcastExchange (28) - +- * ColumnarToRow (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- BroadcastExchange (29) + +- * CometColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] @@ -65,9 +73,12 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Arguments: [s_store_sk#12, s_number_employees#13, s_city#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] @@ -90,9 +101,12 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] @@ -115,53 +129,58 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -(21) ColumnarToRow [codegen id : 1] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] - -(22) HashAggregate [codegen id : 1] +(21) CometHashAggregate Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#18, sum#19] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -(23) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(22) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(24) HashAggregate [codegen id : 3] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] +(23) CometColumnarToRow [codegen id : 2] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] + +(24) HashAggregate [codegen id : 2] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#22, sum(UnscaledValue(ss_net_profit#7))#23] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#22,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#23,17,2) AS profit#25] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#20, sum(UnscaledValue(ss_net_profit#7))#21] +Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#20,17,2) AS amt#22, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#21,17,2) AS profit#23] -(25) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Arguments: [c_customer_sk#26, c_first_name#27, c_last_name#28] +(25) CometNativeScan parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (26) CometFilter -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Condition : isnotnull(c_customer_sk#26) +Input [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Condition : isnotnull(c_customer_sk#24) + +(27) CometProject +Input [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#25, 20)) AS c_first_name#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#26, 30)) AS c_last_name#28] -(27) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [c_customer_sk#24, c_first_name#27, c_last_name#28] -(28) BroadcastExchange -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(29) BroadcastExchange +Input [3]: [c_customer_sk#24, c_first_name#27, c_last_name#28] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(29) BroadcastHashJoin [codegen id : 3] +(30) BroadcastHashJoin [codegen id : 2] Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#26] +Right keys [1]: [c_customer_sk#24] Join type: Inner Join condition: None -(30) Project [codegen id : 3] -Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#24, profit#25, c_customer_sk#26, c_first_name#27, c_last_name#28] +(31) Project [codegen id : 2] +Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23, s_city#14] +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#22, profit#23, c_customer_sk#24, c_first_name#27, c_last_name#28] -(31) TakeOrderedAndProject -Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#25 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25] +(32) TakeOrderedAndProject +Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23, s_city#14] +Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#23 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/extended.txt new file mode 100644 index 0000000000..182f86474f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/extended.txt @@ -0,0 +1,34 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_ticket_number#1, ss_customer_sk#2, s_city#3, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#4))#5,17,2) AS amt#6, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#8,17,2) AS profit#9)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 25 out of 30 eligible operators (83%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/simplified.txt index 2f53027b64..c1a85b62e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/simplified.txt @@ -1,38 +1,37 @@ TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt] - WholeStageCodegen (3) + WholeStageCodegen (2) Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit] BroadcastHashJoin [ss_customer_sk,c_customer_sk] HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk,s_city] #3 - CometProject [s_store_sk,s_city] - CometFilter [s_store_sk,s_number_employees,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_number_employees,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometColumnarToRow + InputAdapter + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk,s_city] #3 + CometProject [s_store_sk,s_city] + CometFilter [s_store_sk,s_number_employees,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #5 - WholeStageCodegen (2) - ColumnarToRow + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/explain.txt index d4600251d2..f3f2addf83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/explain.txt @@ -1,59 +1,68 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (35) - +- CometHashAggregate (34) - +- CometExchange (33) - +- CometHashAggregate (32) - +- CometBroadcastHashJoin (31) - :- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (14) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometFilter (28) - +- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (17) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (37) + +- CometHashAggregate (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometBroadcastHashJoin (33) + :- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.customer_address (15) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.customer_address (18) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.customer (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] @@ -76,152 +85,169 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) +Condition : (isnotnull(s_store_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#9, 10)))) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#9, 10)) AS s_zip#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [ss_net_profit#2, s_store_name#8, s_zip#9], [ss_net_profit#2, s_store_name#8, s_zip#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [1]: [ca_zip#10] -Arguments: [ca_zip#10] - -(15) CometFilter -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) - -(16) CometProject -Input [1]: [ca_zip#10] -Arguments: [ca_zip#11], [substr(ca_zip#10, 1, 5) AS ca_zip#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#12, ca_zip#13] -Arguments: [ca_address_sk#12, ca_zip#13] - -(18) CometFilter -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) +(14) CometProject +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] -(19) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14, c_preferred_cust_flag#15] +(15) CometNativeScan parquet spark_catalog.default.customer_address +Output [1]: [ca_zip#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +ReadSchema: struct -(20) CometFilter -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) - -(21) CometProject -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14], [c_current_addr_sk#14] - -(22) CometBroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: [c_current_addr_sk#14] - -(23) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#12, ca_zip#13] -Right output [1]: [c_current_addr_sk#14] -Arguments: [ca_address_sk#12], [c_current_addr_sk#14], Inner, BuildRight +(16) CometFilter +Input [1]: [ca_zip#11] +Condition : (substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5))) -(24) CometProject -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] -Arguments: [ca_zip#13], [ca_zip#13] +(17) CometProject +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5) AS ca_zip#12] + +(18) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) + +(20) CometProject +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#14, 10)) AS ca_zip#15] + +(21) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_preferred_cust_flag), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((isnotnull(c_preferred_cust_flag#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#17, 1)) = Y)) AND isnotnull(c_current_addr_sk#16)) + +(23) CometProject +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] + +(24) CometBroadcastExchange +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] + +(25) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight + +(26) CometProject +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] -(25) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +(27) CometHashAggregate +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] -(26) CometExchange -Input [2]: [ca_zip#13, count#16] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(28) CometExchange +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [2]: [ca_zip#13, count#16] -Keys [1]: [ca_zip#13] +(29) CometHashAggregate +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] -(28) CometFilter -Input [2]: [ca_zip#17, cnt#18] -Condition : (cnt#18 > 10) +(30) CometFilter +Input [2]: [ca_zip#19, cnt#20] +Condition : (cnt#20 > 10) -(29) CometProject -Input [2]: [ca_zip#17, cnt#18] -Arguments: [ca_zip#17], [ca_zip#17] +(31) CometProject +Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] -(30) CometBroadcastExchange -Input [1]: [ca_zip#17] -Arguments: [ca_zip#17] +(32) CometBroadcastExchange +Input [1]: [ca_zip#19] +Arguments: [ca_zip#19] -(31) CometBroadcastHashJoin -Left output [1]: [ca_zip#11] -Right output [1]: [ca_zip#17] -Arguments: [coalesce(ca_zip#11, ), isnull(ca_zip#11)], [coalesce(ca_zip#17, ), isnull(ca_zip#17)], LeftSemi, BuildRight +(33) CometBroadcastHashJoin +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight -(32) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(34) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -(33) CometExchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(35) CometExchange +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(36) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -(35) CometBroadcastExchange -Input [1]: [ca_zip#11] -Arguments: [ca_zip#11] +(37) CometBroadcastExchange +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] -(36) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Right output [1]: [ca_zip#11] -Arguments: [substr(s_zip#9, 1, 2)], [substr(ca_zip#11, 1, 2)], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight -(37) CometProject -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] +(39) CometProject +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] -(38) ColumnarToRow [codegen id : 1] -Input [2]: [ss_net_profit#2, s_store_name#8] - -(39) HashAggregate [codegen id : 1] +(40) CometHashAggregate Input [2]: [ss_net_profit#2, s_store_name#8] Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#19] -Results [2]: [s_store_name#8, sum#20] -(40) Exchange -Input [2]: [s_store_name#8, sum#20] -Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(41) CometExchange +Input [2]: [s_store_name#8, sum#21] +Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(42) CometColumnarToRow [codegen id : 1] +Input [2]: [s_store_name#8, sum#21] -(41) HashAggregate [codegen id : 2] -Input [2]: [s_store_name#8, sum#20] +(43) HashAggregate [codegen id : 1] +Input [2]: [s_store_name#8, sum#21] Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#21] -Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#21,17,2) AS sum(ss_net_profit)#22] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS sum(ss_net_profit)#23] -(42) TakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#22] -Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] +(44) TakeOrderedAndProject +Input [2]: [s_store_name#8, sum(ss_net_profit)#23] +Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/extended.txt new file mode 100644 index 0000000000..a9f14e6c2e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/extended.txt @@ -0,0 +1,46 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_name#1, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#3,17,2) AS sum(ss_net_profit)#4)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 41 out of 43 eligible operators (95%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/simplified.txt index 31df269cff..24dbc6594c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/simplified.txt @@ -1,46 +1,46 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum] - InputAdapter - Exchange [s_store_name] #1 - WholeStageCodegen (1) - HashAggregate [s_store_name,ss_net_profit] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,s_store_name] - CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] - CometProject [ss_net_profit,s_store_name,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [ca_zip] #4 + CometColumnarToRow + InputAdapter + CometExchange [s_store_name] #1 + CometHashAggregate [ss_net_profit] [s_store_name,sum] + CometProject [ss_net_profit,s_store_name] + CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] + CometProject [ss_net_profit,s_store_name,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [ca_zip] #4 + CometHashAggregate [ca_zip] + CometExchange [ca_zip] #5 CometHashAggregate [ca_zip] - CometExchange [ca_zip] #5 - CometHashAggregate [ca_zip] - CometBroadcastHashJoin [ca_zip,ca_zip] - CometProject [ca_zip] [ca_zip] - CometFilter [ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_zip] - CometBroadcastExchange [ca_zip] #6 - CometProject [ca_zip] - CometFilter [ca_zip,cnt] - CometHashAggregate [ca_zip,cnt,ca_zip,count,count(1)] - CometExchange [ca_zip] #7 - CometHashAggregate [ca_zip,count] - CometProject [ca_zip] - CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] - CometBroadcastExchange [c_current_addr_sk] #8 - CometProject [c_current_addr_sk] - CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_current_addr_sk,c_preferred_cust_flag] + CometBroadcastHashJoin [ca_zip,ca_zip] + CometProject [ca_zip] [ca_zip] + CometFilter [ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_zip] + CometBroadcastExchange [ca_zip] #6 + CometProject [ca_zip] + CometFilter [ca_zip,cnt] + CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] + CometExchange [ca_zip] #7 + CometHashAggregate [ca_zip,count] + CometProject [ca_zip] + CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [c_current_addr_sk] #8 + CometProject [c_current_addr_sk] + CometFilter [c_current_addr_sk,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/explain.txt index 2f3fac41db..8ce4a84e35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/explain.txt @@ -1,79 +1,119 @@ == Physical Plan == -TakeOrderedAndProject (70) -+- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Expand (66) - +- Union (65) - :- * HashAggregate (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- * ColumnarToRow (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometProject (17) - : : : : +- CometBroadcastHashJoin (16) - : : : : :- CometProject (11) - : : : : : +- CometSortMergeJoin (10) - : : : : : :- CometSort (4) - : : : : : : +- CometExchange (3) - : : : : : : +- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometSort (9) - : : : : : +- CometExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : : : +- CometBroadcastExchange (15) - : : : : +- CometProject (14) - : : : : +- CometFilter (13) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - : : : +- CometBroadcastExchange (20) - : : : +- CometFilter (19) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (18) - : : +- CometBroadcastExchange (26) - : : +- CometProject (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (23) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (29) - :- * HashAggregate (62) - : +- Exchange (61) - : +- * HashAggregate (60) - : +- * ColumnarToRow (59) - : +- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometProject (49) - : : : : +- CometBroadcastHashJoin (48) - : : : : :- CometProject (46) - : : : : : +- CometSortMergeJoin (45) - : : : : : :- CometSort (42) - : : : : : : +- CometExchange (41) - : : : : : : +- CometFilter (40) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : +- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- ReusedExchange (47) - : : : +- ReusedExchange (50) - : : +- ReusedExchange (53) - : +- ReusedExchange (56) - +- * HashAggregate (64) - +- ReusedExchange (63) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (106) ++- * HashAggregate (105) + +- * CometColumnarToRow (104) + +- CometColumnarExchange (103) + +- * HashAggregate (102) + +- * Expand (101) + +- Union (100) + :- * HashAggregate (39) + : +- * CometColumnarToRow (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (17) + : : : : +- CometBroadcastHashJoin (16) + : : : : :- CometProject (11) + : : : : : +- CometSortMergeJoin (10) + : : : : : :- CometSort (4) + : : : : : : +- CometExchange (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometSort (9) + : : : : : +- CometExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : : : +- CometBroadcastExchange (15) + : : : : +- CometProject (14) + : : : : +- CometFilter (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : : : +- CometBroadcastExchange (21) + : : : +- CometProject (20) + : : : +- CometFilter (19) + : : : +- CometNativeScan parquet spark_catalog.default.store (18) + : : +- CometBroadcastExchange (27) + : : +- CometProject (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.item (24) + : +- CometBroadcastExchange (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometNativeScan parquet spark_catalog.default.promotion (30) + :- * HashAggregate (69) + : +- * CometColumnarToRow (68) + : +- CometExchange (67) + : +- CometHashAggregate (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (53) + : : : : +- CometBroadcastHashJoin (52) + : : : : :- CometProject (50) + : : : : : +- CometSortMergeJoin (49) + : : : : : :- CometSort (43) + : : : : : : +- CometExchange (42) + : : : : : : +- CometFilter (41) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (40) + : : : : : +- CometSort (48) + : : : : : +- CometExchange (47) + : : : : : +- CometProject (46) + : : : : : +- CometFilter (45) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (44) + : : : : +- ReusedExchange (51) + : : : +- CometBroadcastExchange (57) + : : : +- CometProject (56) + : : : +- CometFilter (55) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (54) + : : +- ReusedExchange (60) + : +- ReusedExchange (63) + +- * HashAggregate (99) + +- * CometColumnarToRow (98) + +- CometExchange (97) + +- CometHashAggregate (96) + +- CometProject (95) + +- CometBroadcastHashJoin (94) + :- CometProject (92) + : +- CometBroadcastHashJoin (91) + : :- CometProject (89) + : : +- CometBroadcastHashJoin (88) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (80) + : : : : +- CometSortMergeJoin (79) + : : : : :- CometSort (73) + : : : : : +- CometExchange (72) + : : : : : +- CometFilter (71) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (70) + : : : : +- CometSort (78) + : : : : +- CometExchange (77) + : : : : +- CometProject (76) + : : : : +- CometFilter (75) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (74) + : : : +- ReusedExchange (81) + : : +- CometBroadcastExchange (87) + : : +- CometProject (86) + : : +- CometFilter (85) + : : +- CometNativeScan parquet spark_catalog.default.web_site (84) + : +- ReusedExchange (90) + +- ReusedExchange (93) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -87,9 +127,12 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (6) CometFilter Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] @@ -116,9 +159,12 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(12) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] +ReadSchema: struct (13) CometFilter Input [2]: [d_date_sk#13, d_date#14] @@ -141,234 +187,407 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(18) CometNativeScan: `spark_catalog`.`default`.`store` +(18) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (19) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(20) CometBroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#16, 16)) AS s_store_id#17] -(21) CometBroadcastHashJoin +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] +Right output [2]: [s_store_sk#15, s_store_id#17] Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(23) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17, i_current_price#18] - -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] - -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] - -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(29) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19, p_channel_tv#20] - -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] - -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] - -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight - -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(35) ColumnarToRow [codegen id : 1] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(36) HashAggregate [codegen id : 1] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) + +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] + +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] + +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(30) CometNativeScan parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((isnotnull(p_channel_tv#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#21, 1)) = N)) AND isnotnull(p_promo_sk#20)) + +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] + +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] + +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight + +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] - -(37) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(38) HashAggregate [codegen id : 2] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#34, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#35, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#36, store channel AS channel#37, concat(store, s_store_id#16) AS id#38] -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(40) CometFilter -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(41) CometExchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(42) CometSort -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45], [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST] - -(43) ReusedExchange [Reuses operator id: 8] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] - -(44) CometSort -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49], [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Right output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cs_item_sk#40, cs_order_number#42], [cr_item_sk#46, cr_order_number#47], LeftOuter +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#30, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#31, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#32, store channel AS channel#33, concat(store, s_store_id#17) AS id#34] + +(40) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct + +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) + +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] + +(44) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) (46) CometProject -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] - -(47) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#50] - -(48) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Right output [1]: [d_date_sk#50] -Arguments: [cs_sold_date_sk#45], [d_date_sk#50], Inner, BuildRight - -(49) CometProject -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#50] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] - -(50) ReusedExchange [Reuses operator id: 20] -Output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] - -(51) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Right output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#39], [cp_catalog_page_sk#51], Inner, BuildRight - -(52) CometProject -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#51, cp_catalog_page_id#52] -Arguments: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52], [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(53) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#53] - -(54) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#40], [i_item_sk#53], Inner, BuildRight - -(55) CometProject -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52], [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(56) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#54] - -(57) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#41], [p_promo_sk#54], Inner, BuildRight - -(58) CometProject -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52], [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(59) ColumnarToRow [codegen id : 3] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(60) HashAggregate [codegen id : 3] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Results [6]: [cp_catalog_page_id#52, sum#60, sum#61, isEmpty#62, sum#63, isEmpty#64] - -(61) Exchange -Input [6]: [cp_catalog_page_id#52, sum#60, sum#61, isEmpty#62, sum#63, isEmpty#64] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(62) HashAggregate [codegen id : 4] -Input [6]: [cp_catalog_page_id#52, sum#60, sum#61, isEmpty#62, sum#63, isEmpty#64] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#65, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#66, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#67] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#65,17,2) AS sales#68, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#66 AS returns#69, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#67 AS profit#70, catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#52) AS id#72] - -(63) ReusedExchange [Reuses operator id: 37] -Output [6]: [web_site_id#73, sum#74, sum#75, isEmpty#76, sum#77, isEmpty#78] - -(64) HashAggregate [codegen id : 6] -Input [6]: [web_site_id#73, sum#74, sum#75, isEmpty#76, sum#77, isEmpty#78] -Keys [1]: [web_site_id#73] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#79)), sum(coalesce(cast(wr_return_amt#80 as decimal(12,2)), 0.00)), sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#82 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#79))#83, sum(coalesce(cast(wr_return_amt#80 as decimal(12,2)), 0.00))#84, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#82 as decimal(12,2)), 0.00)))#85] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#79))#83,17,2) AS sales#86, sum(coalesce(cast(wr_return_amt#80 as decimal(12,2)), 0.00))#84 AS returns#87, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#82 as decimal(12,2)), 0.00)))#85 AS profit#88, web channel AS channel#89, concat(web_site, web_site_id#73) AS id#90] - -(65) Union - -(66) Expand [codegen id : 7] -Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] -Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#91, id#92, spark_grouping_id#93] - -(67) HashAggregate [codegen id : 7] -Input [6]: [sales#34, returns#35, profit#36, channel#91, id#92, spark_grouping_id#93] -Keys [3]: [channel#91, id#92, spark_grouping_id#93] -Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] -Aggregate Attributes [6]: [sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Results [9]: [channel#91, id#92, spark_grouping_id#93, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(68) Exchange -Input [9]: [channel#91, id#92, spark_grouping_id#93, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Arguments: hashpartitioning(channel#91, id#92, spark_grouping_id#93, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(69) HashAggregate [codegen id : 8] -Input [9]: [channel#91, id#92, spark_grouping_id#93, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [3]: [channel#91, id#92, spark_grouping_id#93] -Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] -Aggregate Attributes [3]: [sum(sales#34)#106, sum(returns#35)#107, sum(profit#36)#108] -Results [5]: [channel#91, id#92, sum(sales#34)#106 AS sales#109, sum(returns#35)#107 AS returns#110, sum(profit#36)#108 AS profit#111] - -(70) TakeOrderedAndProject -Input [5]: [channel#91, id#92, sales#109, returns#110, profit#111] -Arguments: 100, [channel#91 ASC NULLS FIRST, id#92 ASC NULLS FIRST], [channel#91, id#92, sales#109, returns#110, profit#111] +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] + +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] + +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter + +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] + +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] + +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight + +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] + +(54) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#49, 16)) AS cp_catalog_page_id#50] + +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] + +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight + +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] + +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight + +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] + +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight + +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] + +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] + +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#62, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#63, catalog channel AS channel#64, concat(catalog_page, cp_catalog_page_id#50) AS id#65] + +(70) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) + +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] + +(74) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] +ReadSchema: struct + +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) + +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] + +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] + +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter + +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] + +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] + +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight + +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] + +(84) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) + +(86) CometProject +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#80, 16)) AS web_site_id#81] + +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] + +(88) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight + +(89) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] + +(91) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight + +(92) CometProject +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#92, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#93, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#81) AS id#96] + +(100) Union + +(101) Expand [codegen id : 4] +Input [5]: [sales#30, returns#31, profit#32, channel#33, id#34] +Arguments: [[sales#30, returns#31, profit#32, channel#33, id#34, 0], [sales#30, returns#31, profit#32, channel#33, null, 1], [sales#30, returns#31, profit#32, null, null, 3]], [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] + +(102) HashAggregate [codegen id : 4] +Input [6]: [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [partial_sum(sales#30), partial_sum(returns#31), partial_sum(profit#32)] +Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] +Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(103) CometColumnarExchange +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(104) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(105) HashAggregate [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [sum(sales#30), sum(returns#31), sum(profit#32)] +Aggregate Attributes [3]: [sum(sales#30)#112, sum(returns#31)#113, sum(profit#32)#114] +Results [5]: [channel#97, id#98, sum(sales#30)#112 AS sales#115, sum(returns#31)#113 AS returns#116, sum(profit#32)#114 AS profit#117] + +(106) TakeOrderedAndProject +Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] +Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/extended.txt new file mode 100644 index 0000000000..990f64b348 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/extended.txt @@ -0,0 +1,126 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#1))#2,17,2) AS sales#3, sum(coalesce(cast(sr_return_amt#4 as decimal(12,2)), 0.00))#5 AS returns#6, sum((ss_net_profit#7 - coalesce(cast(sr_net_loss#8 as decimal(12,2)), 0.00)))#9 AS profit#10, store channel AS channel#11, concat(store, s_store_id#12) AS id#13)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#14))#15,17,2) AS sales#16, sum(coalesce(cast(cr_return_amount#17 as decimal(12,2)), 0.00))#18 AS returns#19, sum((cs_net_profit#20 - coalesce(cast(cr_net_loss#21 as decimal(12,2)), 0.00)))#22 AS profit#23, catalog channel AS channel#24, concat(catalog_page, cp_catalog_page_id#25) AS id#26)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#27))#28,17,2) AS sales#29, sum(coalesce(cast(wr_return_amt#30 as decimal(12,2)), 0.00))#31 AS returns#32, sum((ws_net_profit#33 - coalesce(cast(wr_net_loss#34 as decimal(12,2)), 0.00)))#35 AS profit#36, web channel AS channel#37, concat(web_site, web_site_id#38) AS id#39)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_site + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 112 out of 120 eligible operators (93%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/simplified.txt index 386f58333c..c5cc263e59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/simplified.txt @@ -1,84 +1,116 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (8) + WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (1) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #4 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #4 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #7 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #8 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_tv] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #9 - WholeStageCodegen (3) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #10 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #4 - ReusedExchange [d_date_sk] #5 - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 - ReusedExchange [i_item_sk] #7 - ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [web_site_id,sum,sum,isEmpty,sum,isEmpty] #2 + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #8 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #9 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #10 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #11 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #13 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #14 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #15 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/explain.txt index c412d20fc7..1e92c1ff76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/explain.txt @@ -1,67 +1,78 @@ == Physical Plan == -TakeOrderedAndProject (50) -+- * Project (49) - +- * BroadcastHashJoin Inner BuildRight (48) - :- * Project (43) - : +- * BroadcastHashJoin Inner BuildRight (42) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (18) - : : : +- * HashAggregate (17) - : : : +- Exchange (16) - : : : +- * HashAggregate (15) - : : : +- * ColumnarToRow (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (54) ++- * Project (53) + +- * BroadcastHashJoin Inner BuildRight (52) + :- * Project (46) + : +- * BroadcastHashJoin Inner BuildRight (45) + : :- * Project (39) + : : +- * BroadcastHashJoin Inner BuildRight (38) + : : :- * Filter (19) + : : : +- * HashAggregate (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- BroadcastExchange (35) - : : +- * Filter (34) - : : +- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * ColumnarToRow (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometFilter (20) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (19) - : : : +- ReusedExchange (21) - : : +- ReusedExchange (24) - : +- BroadcastExchange (41) - : +- * ColumnarToRow (40) - : +- CometFilter (39) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (38) - +- BroadcastExchange (47) - +- * ColumnarToRow (46) - +- CometFilter (45) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : : +- BroadcastExchange (37) + : : +- * Filter (36) + : : +- * HashAggregate (35) + : : +- * CometColumnarToRow (34) + : : +- CometColumnarExchange (33) + : : +- * HashAggregate (32) + : : +- * HashAggregate (31) + : : +- * CometColumnarToRow (30) + : : +- CometExchange (29) + : : +- CometHashAggregate (28) + : : +- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (21) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (20) + : : : +- ReusedExchange (22) + : : +- ReusedExchange (25) + : +- BroadcastExchange (44) + : +- * CometColumnarToRow (43) + : +- CometProject (42) + : +- CometFilter (41) + : +- CometNativeScan parquet spark_catalog.default.customer (40) + +- BroadcastExchange (51) + +- * CometColumnarToRow (50) + +- CometProject (49) + +- CometFilter (48) + +- CometNativeScan parquet spark_catalog.default.customer_address (47) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#4)] +PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_year#6] @@ -84,192 +95,216 @@ Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)))) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)) AS ca_state#9] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#7, ca_state#8] +Right output [2]: [ca_address_sk#7, ca_state#9] Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] - -(14) ColumnarToRow [codegen id : 1] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] +(14) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] -(15) HashAggregate [codegen id : 1] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -(16) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) HashAggregate [codegen id : 7] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(17) CometColumnarToRow [codegen id : 5] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] + +(18) HashAggregate [codegen id : 5] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] +Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#9 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] -(18) Filter [codegen id : 7] +(19) Filter [codegen id : 5] Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] Condition : isnotnull(ctr_total_return#14) -(19) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(20) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] -Arguments: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#18)] +PushedFilters: [IsNotNull(cr_returning_addr_sk)] +ReadSchema: struct -(20) CometFilter +(21) CometFilter Input [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] Condition : isnotnull(cr_returning_addr_sk#16) -(21) ReusedExchange [Reuses operator id: 6] +(22) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#19] -(22) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] Right output [1]: [d_date_sk#19] Arguments: [cr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight -(23) CometProject +(24) CometProject Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18, d_date_sk#19] Arguments: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17], [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] -(24) ReusedExchange [Reuses operator id: 11] +(25) ReusedExchange [Reuses operator id: 12] Output [2]: [ca_address_sk#20, ca_state#21] -(25) CometBroadcastHashJoin +(26) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] Right output [2]: [ca_address_sk#20, ca_state#21] Arguments: [cr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(26) CometProject +(27) CometProject Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, ca_address_sk#20, ca_state#21] Arguments: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21], [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] -(27) ColumnarToRow [codegen id : 2] -Input [3]: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] - -(28) HashAggregate [codegen id : 2] +(28) CometHashAggregate Input [3]: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] Keys [2]: [cr_returning_customer_sk#15, ca_state#21] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#17))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [cr_returning_customer_sk#15, ca_state#21, sum#23] -(29) Exchange -Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#23] -Arguments: hashpartitioning(cr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(29) CometExchange +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] +Arguments: hashpartitioning(cr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] -(30) HashAggregate [codegen id : 3] -Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#23] +(31) HashAggregate [codegen id : 1] +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] Keys [2]: [cr_returning_customer_sk#15, ca_state#21] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))] Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))#11] -Results [2]: [ca_state#21 AS ctr_state#24, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#17))#11,17,2) AS ctr_total_return#25] - -(31) HashAggregate [codegen id : 3] -Input [2]: [ctr_state#24, ctr_total_return#25] -Keys [1]: [ctr_state#24] -Functions [1]: [partial_avg(ctr_total_return#25)] -Aggregate Attributes [2]: [sum#26, count#27] -Results [3]: [ctr_state#24, sum#28, count#29] - -(32) Exchange -Input [3]: [ctr_state#24, sum#28, count#29] -Arguments: hashpartitioning(ctr_state#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(33) HashAggregate [codegen id : 4] -Input [3]: [ctr_state#24, sum#28, count#29] -Keys [1]: [ctr_state#24] -Functions [1]: [avg(ctr_total_return#25)] -Aggregate Attributes [1]: [avg(ctr_total_return#25)#30] -Results [2]: [(avg(ctr_total_return#25)#30 * 1.2) AS (avg(ctr_total_return) * 1.2)#31, ctr_state#24] - -(34) Filter [codegen id : 4] -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#31) - -(35) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] +Results [2]: [ca_state#21 AS ctr_state#23, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#17))#11,17,2) AS ctr_total_return#24] + +(32) HashAggregate [codegen id : 1] +Input [2]: [ctr_state#23, ctr_total_return#24] +Keys [1]: [ctr_state#23] +Functions [1]: [partial_avg(ctr_total_return#24)] +Aggregate Attributes [2]: [sum#25, count#26] +Results [3]: [ctr_state#23, sum#27, count#28] + +(33) CometColumnarExchange +Input [3]: [ctr_state#23, sum#27, count#28] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(34) CometColumnarToRow [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] + +(35) HashAggregate [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] +Keys [1]: [ctr_state#23] +Functions [1]: [avg(ctr_total_return#24)] +Aggregate Attributes [1]: [avg(ctr_total_return#24)#29] +Results [2]: [(avg(ctr_total_return#24)#29 * 1.2) AS (avg(ctr_total_return) * 1.2)#30, ctr_state#23] + +(36) Filter [codegen id : 2] +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#30) + +(37) BroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] -(36) BroadcastHashJoin [codegen id : 7] +(38) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#24] +Right keys [1]: [ctr_state#23] Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) +Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#30) -(37) Project [codegen id : 7] +(39) Project [codegen id : 5] Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#31, ctr_state#24] +Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#30, ctr_state#23] -(38) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Arguments: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] +(40) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct -(39) CometFilter -Input [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_current_addr_sk#34)) +(41) CometFilter +Input [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Condition : (isnotnull(c_customer_sk#31) AND isnotnull(c_current_addr_sk#33)) -(40) ColumnarToRow [codegen id : 5] -Input [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] +(42) CometProject +Input [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Arguments: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40], [c_customer_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#32, 16)) AS c_customer_id#37, c_current_addr_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#34, 10)) AS c_salutation#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#35, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#36, 30)) AS c_last_name#40] -(41) BroadcastExchange -Input [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(43) CometColumnarToRow [codegen id : 3] +Input [6]: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] -(42) BroadcastHashJoin [codegen id : 7] +(44) BroadcastExchange +Input [6]: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] + +(45) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#32] +Right keys [1]: [c_customer_sk#31] Join type: Inner Join condition: None -(43) Project [codegen id : 7] -Output [6]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] +(46) Project [codegen id : 5] +Output [6]: [ctr_total_return#14, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] +Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] + +(47) CometNativeScan parquet spark_catalog.default.customer_address +Output [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct -(44) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Arguments: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] +(48) CometFilter +Input [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Condition : ((isnotnull(ca_state#48) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#48, 2)) = GA)) AND isnotnull(ca_address_sk#41)) -(45) CometFilter -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Condition : ((isnotnull(ca_state#45) AND (ca_state#45 = GA)) AND isnotnull(ca_address_sk#38)) +(49) CometProject +Input [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Arguments: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58], [ca_address_sk#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#42, 10)) AS ca_street_number#53, ca_street_name#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_type#44, 15)) AS ca_street_type#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_suite_number#45, 10)) AS ca_suite_number#55, ca_city#46, ca_county#47, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#48, 2)) AS ca_state#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#49, 10)) AS ca_zip#57, ca_country#50, ca_gmt_offset#51, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_location_type#52, 20)) AS ca_location_type#58] -(46) ColumnarToRow [codegen id : 6] -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] +(50) CometColumnarToRow [codegen id : 4] +Input [12]: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] -(47) BroadcastExchange -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(51) BroadcastExchange +Input [12]: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(48) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#38] +(52) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#33] +Right keys [1]: [ca_address_sk#41] Join type: Inner Join condition: None -(49) Project [codegen id : 7] -Output [16]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49, ctr_total_return#14] -Input [18]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] +(53) Project [codegen id : 5] +Output [16]: [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] +Input [18]: [ctr_total_return#14, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40, ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] -(50) TakeOrderedAndProject -Input [16]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49, ctr_total_return#14] -Arguments: 100, [c_customer_id#33 ASC NULLS FIRST, c_salutation#35 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, c_last_name#37 ASC NULLS FIRST, ca_street_number#39 ASC NULLS FIRST, ca_street_name#40 ASC NULLS FIRST, ca_street_type#41 ASC NULLS FIRST, ca_suite_number#42 ASC NULLS FIRST, ca_city#43 ASC NULLS FIRST, ca_county#44 ASC NULLS FIRST, ca_state#45 ASC NULLS FIRST, ca_zip#46 ASC NULLS FIRST, ca_country#47 ASC NULLS FIRST, ca_gmt_offset#48 ASC NULLS FIRST, ca_location_type#49 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49, ctr_total_return#14] +(54) TakeOrderedAndProject +Input [16]: [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] +Arguments: 100, [c_customer_id#37 ASC NULLS FIRST, c_salutation#38 ASC NULLS FIRST, c_first_name#39 ASC NULLS FIRST, c_last_name#40 ASC NULLS FIRST, ca_street_number#53 ASC NULLS FIRST, ca_street_name#43 ASC NULLS FIRST, ca_street_type#54 ASC NULLS FIRST, ca_suite_number#55 ASC NULLS FIRST, ca_city#46 ASC NULLS FIRST, ca_county#47 ASC NULLS FIRST, ca_state#56 ASC NULLS FIRST, ca_zip#57 ASC NULLS FIRST, ca_country#50 ASC NULLS FIRST, ca_gmt_offset#51 ASC NULLS FIRST, ca_location_type#58 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/extended.txt new file mode 100644 index 0000000000..a86628a730 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/extended.txt @@ -0,0 +1,62 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Filter + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cr_returning_customer_sk#1 AS ctr_customer_sk#2, ca_state#3 AS ctr_state#4, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#5))#6,17,2) AS ctr_total_return#7)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- BroadcastExchange + : : +- Filter + : : +- HashAggregate + : : +- CometColumnarToRow + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ca_state#8 AS ctr_state#9, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#10))#6,17,2) AS ctr_total_return#11)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 39 out of 55 eligible operators (70%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/simplified.txt index e5c7a02f63..b864e77c7f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - WholeStageCodegen (7) + WholeStageCodegen (5) Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] BroadcastHashJoin [c_current_addr_sk,ca_address_sk] Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] @@ -8,60 +8,60 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] Filter [ctr_total_return] HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (1) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometColumnarToRow + InputAdapter + CometExchange [cr_returning_customer_sk,ca_state] #1 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #4 - WholeStageCodegen (4) + WholeStageCodegen (2) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (3) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (2) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - ReusedExchange [ca_address_sk,ca_state] #3 + CometColumnarToRow + InputAdapter + CometColumnarExchange [ctr_state] #5 + WholeStageCodegen (1) + HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] + HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [cr_returning_customer_sk,ca_state] #6 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 InputAdapter BroadcastExchange #7 - WholeStageCodegen (5) - ColumnarToRow + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] InputAdapter BroadcastExchange #8 - WholeStageCodegen (6) - ColumnarToRow + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/explain.txt index fdcea86558..870b70e457 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (26) +* CometColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) +- CometExchange (23) @@ -13,23 +13,26 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) + : : +- CometNativeScan parquet spark_catalog.default.inventory (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) +- CometProject (19) +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (17) + +- CometNativeScan parquet spark_catalog.default.store_sales (17) -(1) CometNativeScan: `spark_catalog`.`default`.`item` +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,423,821]), IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] @@ -37,101 +40,111 @@ Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) A (3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#2, 16)) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#9)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] +ReadSchema: struct (5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) (6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] (7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] (8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight (9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +(10) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] +ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11, ss_sold_date_sk#12] +(17) CometNativeScan parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (18) CometFilter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) (19) CometProject -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11], [ss_item_sk#11] +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] (20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#11] -Arguments: [i_item_sk#1], [ss_item_sk#11], Inner, BuildLeft +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/extended.txt new file mode 100644 index 0000000000..6e00d25e4c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.inventory + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_sales + +Comet accelerated 25 out of 25 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/simplified.txt index b8535baaf2..4deaf58d7e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] @@ -12,17 +12,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [ss_item_sk] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/explain.txt index b71044ea66..f06dbc74e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/explain.txt @@ -1,192 +1,317 @@ == Physical Plan == -TakeOrderedAndProject (35) -+- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Project (31) - : +- * BroadcastHashJoin Inner BuildRight (30) - : :- * HashAggregate (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- * ColumnarToRow (23) - : : +- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometFilter (12) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) - : +- BroadcastExchange (29) - : +- * HashAggregate (28) - : +- ReusedExchange (27) - +- ReusedExchange (32) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` +* CometColumnarToRow (56) ++- CometTakeOrderedAndProject (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometNativeScan parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (38) + : +- CometHashAggregate (37) + : +- CometExchange (36) + : +- CometHashAggregate (35) + : +- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- ReusedExchange (32) + +- CometBroadcastExchange (52) + +- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) + + +(1) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Arguments: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#3)] +PushedFilters: [IsNotNull(sr_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Condition : (isnotnull(i_item_sk#4) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)))) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#6] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] + +(7) CometBroadcastHashJoin Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] +Right output [2]: [i_item_sk#4, i_item_id#6] Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6, d_date#7] - -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8, d_week_seq#9] - -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#10, d_week_seq#11] -Arguments: [d_date#10, d_week_seq#11] - -(12) CometFilter -Input [2]: [d_date#10, d_week_seq#11] -Condition : d_date#10 IN (2000-06-30,2000-09-27,2000-11-17) - -(13) CometProject -Input [2]: [d_date#10, d_week_seq#11] -Arguments: [d_week_seq#11], [d_week_seq#11] - -(14) CometBroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: [d_week_seq#11] - -(15) CometBroadcastHashJoin -Left output [2]: [d_date#8, d_week_seq#9] -Right output [1]: [d_week_seq#11] -Arguments: [d_week_seq#9], [d_week_seq#11], LeftSemi, BuildRight - -(16) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] - -(17) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] - -(18) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight - -(19) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(21) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#3], [d_date_sk#6], Inner, BuildRight - -(22) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [sr_return_quantity#2, i_item_id#5], [sr_return_quantity#2, i_item_id#5] - -(23) ColumnarToRow [codegen id : 1] -Input [2]: [sr_return_quantity#2, i_item_id#5] - -(24) HashAggregate [codegen id : 1] -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(8) CometProject +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) + +(11) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(12) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#11, d_week_seq#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_date, [2000-06-30,2000-09-27,2000-11-17])] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date#11, d_week_seq#12] +Condition : d_date#11 IN (2000-06-30,2000-09-27,2000-11-17) + +(14) CometProject +Input [2]: [d_date#11, d_week_seq#12] +Arguments: [d_week_seq#12], [d_week_seq#12] + +(15) CometBroadcastExchange +Input [1]: [d_week_seq#12] +Arguments: [d_week_seq#12] + +(16) CometBroadcastHashJoin +Left output [2]: [d_date#9, d_week_seq#10] +Right output [1]: [d_week_seq#12] +Arguments: [d_week_seq#10], [d_week_seq#12], LeftSemi, BuildRight + +(17) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] + +(18) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] + +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight + +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] -(25) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(25) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum(sr_return_quantity#2)#14] -Results [2]: [i_item_id#5 AS item_id#15, sum(sr_return_quantity#2)#14 AS sr_item_qty#16] - -(27) ReusedExchange [Reuses operator id: 25] -Output [2]: [i_item_id#17, sum#18] - -(28) HashAggregate [codegen id : 3] -Input [2]: [i_item_id#17, sum#18] -Keys [1]: [i_item_id#17] -Functions [1]: [sum(cr_return_quantity#19)] -Aggregate Attributes [1]: [sum(cr_return_quantity#19)#20] -Results [2]: [i_item_id#17 AS item_id#21, sum(cr_return_quantity#19)#20 AS cr_item_qty#22] -(29) BroadcastExchange +(27) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#16)] +PushedFilters: [IsNotNull(cr_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Condition : isnotnull(cr_item_sk#14) + +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#17, i_item_id#18] + +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Right output [2]: [i_item_sk#17, i_item_id#18] +Arguments: [cr_item_sk#14], [i_item_sk#17], Inner, BuildRight + +(31) CometProject +Input [5]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16, i_item_sk#17, i_item_id#18] +Arguments: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18], [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] + +(32) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#19] + +(33) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#16], [d_date_sk#19], Inner, BuildRight + +(34) CometProject +Input [4]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18, d_date_sk#19] +Arguments: [cr_return_quantity#15, i_item_id#18], [cr_return_quantity#15, i_item_id#18] + +(35) CometHashAggregate +Input [2]: [cr_return_quantity#15, i_item_id#18] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(cr_return_quantity#15)] + +(36) CometExchange +Input [2]: [i_item_id#18, sum#20] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(37) CometHashAggregate +Input [2]: [i_item_id#18, sum#20] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(cr_return_quantity#15)] + +(38) CometBroadcastExchange Input [2]: [item_id#21, cr_item_qty#22] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#21] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [3]: [item_id#15, sr_item_qty#16, cr_item_qty#22] -Input [4]: [item_id#15, sr_item_qty#16, item_id#21, cr_item_qty#22] - -(32) ReusedExchange [Reuses operator id: 29] -Output [2]: [item_id#23, wr_item_qty#24] - -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#23] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 6] -Output [8]: [item_id#15, sr_item_qty#16, (((cast(sr_item_qty#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as double)))) / 3.0) * 100.0) AS sr_dev#25, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as double)))) / 3.0) * 100.0) AS cr_dev#26, wr_item_qty#24, (((cast(wr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as double)))) / 3.0) * 100.0) AS wr_dev#27, (cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as decimal(20,0)) / 3.0) AS average#28] -Input [5]: [item_id#15, sr_item_qty#16, cr_item_qty#22, item_id#23, wr_item_qty#24] - -(35) TakeOrderedAndProject -Input [8]: [item_id#15, sr_item_qty#16, sr_dev#25, cr_item_qty#22, cr_dev#26, wr_item_qty#24, wr_dev#27, average#28] -Arguments: 100, [item_id#15 ASC NULLS FIRST, sr_item_qty#16 ASC NULLS FIRST], [item_id#15, sr_item_qty#16, sr_dev#25, cr_item_qty#22, cr_dev#26, wr_item_qty#24, wr_dev#27, average#28] +Arguments: [item_id#21, cr_item_qty#22] + +(39) CometBroadcastHashJoin +Left output [2]: [item_id#23, sr_item_qty#24] +Right output [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#23], [item_id#21], Inner, BuildRight + +(40) CometProject +Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] +Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] + +(41) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#27)] +PushedFilters: [IsNotNull(wr_item_sk)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Condition : isnotnull(wr_item_sk#25) + +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#28, i_item_id#29] + +(44) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Right output [2]: [i_item_sk#28, i_item_id#29] +Arguments: [wr_item_sk#25], [i_item_sk#28], Inner, BuildRight + +(45) CometProject +Input [5]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29], [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] + +(46) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#30] + +(47) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] +Right output [1]: [d_date_sk#30] +Arguments: [wr_returned_date_sk#27], [d_date_sk#30], Inner, BuildRight + +(48) CometProject +Input [4]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [wr_return_quantity#26, i_item_id#29], [wr_return_quantity#26, i_item_id#29] + +(49) CometHashAggregate +Input [2]: [wr_return_quantity#26, i_item_id#29] +Keys [1]: [i_item_id#29] +Functions [1]: [partial_sum(wr_return_quantity#26)] + +(50) CometExchange +Input [2]: [i_item_id#29, sum#31] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(51) CometHashAggregate +Input [2]: [i_item_id#29, sum#31] +Keys [1]: [i_item_id#29] +Functions [1]: [sum(wr_return_quantity#26)] + +(52) CometBroadcastExchange +Input [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#32, wr_item_qty#33] + +(53) CometBroadcastHashJoin +Left output [3]: [item_id#23, sr_item_qty#24, cr_item_qty#22] +Right output [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#23], [item_id#32], Inner, BuildRight + +(54) CometProject +Input [5]: [item_id#23, sr_item_qty#24, cr_item_qty#22, item_id#32, wr_item_qty#33] +Arguments: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#23, sr_item_qty#24, (((cast(sr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] + +(55) CometTakeOrderedAndProject +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#23 ASC NULLS FIRST,sr_item_qty#24 ASC NULLS FIRST], output=[item_id#23,sr_item_qty#24,sr_dev#34,cr_item_qty#22,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#23 ASC NULLS FIRST, sr_item_qty#24 ASC NULLS FIRST], [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] + +(56) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/extended.txt new file mode 100644 index 0000000000..742a696f6f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/extended.txt @@ -0,0 +1,88 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 85 out of 85 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/simplified.txt index 81c23c07c4..b787401379 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/simplified.txt @@ -1,43 +1,58 @@ -TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] - WholeStageCodegen (6) - Project [item_id,sr_item_qty,cr_item_qty,wr_item_qty] - BroadcastHashJoin [item_id,item_id] - Project [item_id,sr_item_qty,cr_item_qty] - BroadcastHashJoin [item_id,item_id] - HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,sr_return_quantity] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_return_quantity,i_item_id] - CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] - CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] - CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] - CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [d_date] #4 - CometProject [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - CometBroadcastExchange [d_week_seq] #5 - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum] - InputAdapter - ReusedExchange [i_item_id,sum] #1 - InputAdapter - ReusedExchange [item_id,wr_item_qty] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometBroadcastHashJoin [item_id,sr_item_qty,cr_item_qty,item_id,wr_item_qty] + CometProject [item_id,sr_item_qty,cr_item_qty] + CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] + CometHashAggregate [sum] [item_id,sr_item_qty,i_item_id,sum(sr_return_quantity)] + CometExchange [i_item_id] #1 + CometHashAggregate [sr_return_quantity] [i_item_id,sum] + CometProject [sr_return_quantity,i_item_id] + CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] + CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id] #2 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #4 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #5 + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cr_item_qty] #6 + CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] + CometExchange [i_item_id] #7 + CometHashAggregate [cr_return_quantity] [i_item_id,sum] + CometProject [cr_return_quantity,i_item_id] + CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] + CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [item_id,wr_item_qty] #8 + CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] + CometExchange [i_item_id] #9 + CometHashAggregate [wr_return_quantity] [i_item_id,sum] + CometProject [wr_return_quantity,i_item_id] + CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] + CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/explain.txt index 37a2d3f738..31bc59394a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/explain.txt @@ -1,167 +1,190 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * ColumnarToRow (30) - +- CometBroadcastHashJoin (29) - :- CometBroadcastExchange (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (9) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (14) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`income_band` (19) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (10) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.income_band (20) + +- CometProject (29) + +- CometFilter (28) + +- CometNativeScan parquet spark_catalog.default.store_returns (27) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7, ca_city#8] - -(4) CometFilter -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) - -(5) CometProject -Input [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7], [ca_address_sk#7] - -(6) CometBroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: [ca_address_sk#7] - -(7) CometBroadcastHashJoin -Left output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Right output [1]: [ca_address_sk#7] -Arguments: [c_current_addr_sk#4], [ca_address_sk#7], Inner, BuildRight - -(8) CometProject -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6], [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(10) CometFilter -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) - -(11) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(12) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Right output [1]: [cd_demo_sk#9] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(13) CometProject -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(15) CometFilter -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) - -(16) CometBroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#10], Inner, BuildRight - -(18) CometProject -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] - -(19) CometNativeScan: `spark_catalog`.`default`.`income_band` -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(20) CometFilter -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) - -(21) CometProject -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12], [ib_income_band_sk#12] - -(22) CometBroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: [ib_income_band_sk#12] - -(23) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Right output [1]: [ib_income_band_sk#12] -Arguments: [hd_income_band_sk#11], [ib_income_band_sk#12], Inner, BuildRight - -(24) CometProject -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(25) CometBroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15, sr_returned_date_sk#16] - -(27) CometFilter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) - -(28) CometProject -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15], [sr_cdemo_sk#15] - -(29) CometBroadcastHashJoin -Left output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [1]: [sr_cdemo_sk#15] -Arguments: [cd_demo_sk#9], [sr_cdemo_sk#15], Inner, BuildLeft - -(30) ColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(31) Project [codegen id : 1] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(32) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] +(3) CometProject +Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#1, 16)) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#5, 20)) AS c_first_name#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#6, 30)) AS c_last_name#9] + +(4) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#10, ca_city#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [ca_address_sk#10, ca_city#11] +Condition : ((isnotnull(ca_city#11) AND (ca_city#11 = Edgewood)) AND isnotnull(ca_address_sk#10)) + +(6) CometProject +Input [2]: [ca_address_sk#10, ca_city#11] +Arguments: [ca_address_sk#10], [ca_address_sk#10] + +(7) CometBroadcastExchange +Input [1]: [ca_address_sk#10] +Arguments: [ca_address_sk#10] + +(8) CometBroadcastHashJoin +Left output [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9] +Right output [1]: [ca_address_sk#10] +Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight + +(9) CometProject +Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] + +(10) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(11) CometFilter +Input [1]: [cd_demo_sk#12] +Condition : isnotnull(cd_demo_sk#12) + +(12) CometBroadcastExchange +Input [1]: [cd_demo_sk#12] +Arguments: [cd_demo_sk#12] + +(13) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] +Right output [1]: [cd_demo_sk#12] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight + +(14) CometProject +Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(15) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Condition : (isnotnull(hd_demo_sk#13) AND isnotnull(hd_income_band_sk#14)) + +(17) CometBroadcastExchange +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [hd_demo_sk#13, hd_income_band_sk#14] + +(18) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight + +(19) CometProject +Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] + +(20) CometNativeScan parquet spark_catalog.default.income_band +Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Condition : ((((isnotnull(ib_lower_bound#16) AND isnotnull(ib_upper_bound#17)) AND (ib_lower_bound#16 >= 38128)) AND (ib_upper_bound#17 <= 88128)) AND isnotnull(ib_income_band_sk#15)) + +(22) CometProject +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Arguments: [ib_income_band_sk#15], [ib_income_band_sk#15] + +(23) CometBroadcastExchange +Input [1]: [ib_income_band_sk#15] +Arguments: [ib_income_band_sk#15] + +(24) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] +Right output [1]: [ib_income_band_sk#15] +Arguments: [hd_income_band_sk#14], [ib_income_band_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14, ib_income_band_sk#15] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(26) CometBroadcastExchange +Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(27) CometNativeScan parquet spark_catalog.default.store_returns +Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_cdemo_sk)] +ReadSchema: struct + +(28) CometFilter +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_cdemo_sk#18) + +(29) CometProject +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Arguments: [sr_cdemo_sk#18], [sr_cdemo_sk#18] + +(30) CometBroadcastHashJoin +Left output [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [1]: [sr_cdemo_sk#18] +Arguments: [cd_demo_sk#12], [sr_cdemo_sk#18], Inner, BuildLeft + +(31) CometProject +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] +Arguments: [customer_id#20, customername#21, c_customer_id#7], [c_customer_id#7 AS customer_id#20, concat(c_last_name#9, , , c_first_name#8) AS customername#21, c_customer_id#7] + +(32) CometTakeOrderedAndProject +Input [3]: [customer_id#20, customername#21, c_customer_id#7] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#7 ASC NULLS FIRST], output=[customer_id#20,customername#21]), [customer_id#20, customername#21], 100, 0, [c_customer_id#7 ASC NULLS FIRST], [customer_id#20, customername#21] + +(33) CometColumnarToRow [codegen id : 1] +Input [2]: [customer_id#20, customername#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/extended.txt new file mode 100644 index 0000000000..b025c868d1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_returns + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/simplified.txt index 6fd5331780..2f4baa9f3f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ -TakeOrderedAndProject [c_customer_id,customer_id,customername] - WholeStageCodegen (1) - Project [c_customer_id,c_last_name,c_first_name] - ColumnarToRow - InputAdapter +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customername,c_customer_id] + CometProject [c_last_name,c_first_name] [customer_id,customername,c_customer_id] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk] @@ -13,22 +13,23 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] - CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometNativeScan: `spark_catalog`.`default`.`income_band` [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_cdemo_sk,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/explain.txt index 8f7309cd1c..3f251b7b6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/explain.txt @@ -1,54 +1,61 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) +* CometColumnarToRow (48) ++- CometTakeOrderedAndProject (47) + +- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometBroadcastHashJoin (42) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) : : : : :- CometProject (13) : : : : : +- CometBroadcastHashJoin (12) : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometBroadcastExchange (3) : : : : : : : +- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (9) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometFilter (15) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (14) - : : : +- CometBroadcastExchange (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (19) - : : +- CometBroadcastExchange (27) - : : +- CometProject (26) - : : +- CometFilter (25) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - : +- CometBroadcastExchange (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometProject (16) + : : : : +- CometFilter (15) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (14) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.date_dim (32) + +- CometBroadcastExchange (41) + +- CometProject (40) + +- CometFilter (39) + +- CometNativeScan parquet spark_catalog.default.reason (38) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] +ReadSchema: struct (2) CometFilter Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] @@ -58,9 +65,12 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(4) CometNativeScan parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] +ReadSchema: struct (5) CometFilter Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] @@ -79,9 +89,12 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(9) CometNativeScan: `spark_catalog`.`default`.`web_page` +(9) CometNativeScan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#17] -Arguments: [wp_web_page_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct (10) CometFilter Input [1]: [wp_web_page_sk#17] @@ -100,141 +113,164 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(14) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(14) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (15) CometFilter Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)))) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = 2 yr Degree )))) -(16) CometBroadcastExchange +(16) CometProject Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(17) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight - -(18) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(20) CometFilter -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#22] -(21) CometBroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(17) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(22) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Right output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20], [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23], Inner, BuildRight - -(23) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(18) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight + +(19) CometProject +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] + +(20) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)))) + +(22) CometProject +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)) AS cd_marital_status#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)) AS cd_education_status#27] + +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] + +(24) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25, ca_country#26] +(26) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct -(25) CometFilter -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) +(27) CometFilter +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (IN,OH,NJ) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (WI,CT,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (LA,IA,AR))) -(26) CometProject -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25], [ca_address_sk#24, ca_state#25] +(28) CometProject +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) AS ca_state#31] -(27) CometBroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: [ca_address_sk#24, ca_state#25] +(29) CometBroadcastExchange +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] -(28) CometBroadcastHashJoin +(30) CometBroadcastHashJoin Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [ca_address_sk#24, ca_state#25] -Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#24], Inner, ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight -(29) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] +(31) CometProject +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27, d_year#28] +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#32, d_year#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct -(31) CometFilter -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +(33) CometFilter +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) -(32) CometProject -Input [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27], [d_date_sk#27] +(34) CometProject +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] -(33) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] +(35) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(34) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [1]: [d_date_sk#27] -Arguments: [ws_sold_date_sk#7], [d_date_sk#27], Inner, BuildRight +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight -(35) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] +(37) CometProject +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(36) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] +(38) CometNativeScan parquet spark_catalog.default.reason +Output [2]: [r_reason_sk#34, r_reason_desc#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_sk)] +ReadSchema: struct -(37) CometFilter -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) +(39) CometFilter +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) -(38) CometBroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] +(40) CometProject +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(r_reason_desc#35, 100)) AS r_reason_desc#36] -(39) CometBroadcastHashJoin -Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [wr_reason_sk#12], [r_reason_sk#29], Inner, BuildRight +(41) CometBroadcastExchange +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] -(40) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] -Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] +(42) CometBroadcastHashJoin +Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight -(41) ColumnarToRow [codegen id : 1] -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] +(43) CometProject +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] -(42) HashAggregate [codegen id : 1] -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] +(44) CometHashAggregate +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [6]: [sum#31, count#32, sum#33, count#34, sum#35, count#36] -Results [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -(43) Exchange -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(45) CometExchange +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(44) HashAggregate [codegen id : 2] -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Keys [1]: [r_reason_desc#30] +(46) CometHashAggregate +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [3]: [avg(ws_quantity#4)#43, avg(UnscaledValue(wr_refunded_cash#15))#44, avg(UnscaledValue(wr_fee#14))#45] -Results [4]: [substr(r_reason_desc#30, 1, 20) AS substr(r_reason_desc, 1, 20)#46, avg(ws_quantity#4)#43 AS avg(ws_quantity)#47, cast((avg(UnscaledValue(wr_refunded_cash#15))#44 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#48, cast((avg(UnscaledValue(wr_fee#14))#45 / 100.0) as decimal(11,6)) AS avg(wr_fee)#49] -(45) TakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] -Arguments: 100, [substr(r_reason_desc, 1, 20)#46 ASC NULLS FIRST, avg(ws_quantity)#47 ASC NULLS FIRST, avg(wr_refunded_cash)#48 ASC NULLS FIRST, avg(wr_fee)#49 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] +(47) CometTakeOrderedAndProject +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] + +(48) CometColumnarToRow [codegen id : 1] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/extended.txt new file mode 100644 index 0000000000..358da39c56 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/extended.txt @@ -0,0 +1,50 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 47 out of 47 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/simplified.txt index d970a3cc38..925b2ed9b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/simplified.txt @@ -1,49 +1,50 @@ -TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - WholeStageCodegen (2) - HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count] - InputAdapter - Exchange [r_reason_desc] #1 - WholeStageCodegen (1) - HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] - CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] - CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] - CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 - CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometBroadcastExchange [wp_web_page_sk] #3 - CometFilter [wp_web_page_sk] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #6 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 - CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] + CometHashAggregate [sum,count,sum,count,sum,count] [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] + CometExchange [r_reason_desc] #1 + CometHashAggregate [ws_quantity,wr_refunded_cash,wr_fee] [r_reason_desc,sum,count,sum,count,sum,count] + CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] + CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] + CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] + CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 + CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometBroadcastExchange [wp_web_page_sk] #3 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #6 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 + CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] + CometFilter [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/explain.txt index 6612b949e8..89c12f6044 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/explain.txt @@ -1,40 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (23) -+- * Project (22) - +- Window (21) - +- * Sort (20) - +- Exchange (19) - +- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometExpand (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- CometBroadcastExchange (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +TakeOrderedAndProject (25) ++- * Project (24) + +- Window (23) + +- * CometColumnarToRow (22) + +- CometSort (21) + +- CometColumnarExchange (20) + +- * HashAggregate (19) + +- * CometColumnarToRow (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometExpand (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.item (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -57,69 +66,77 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#7, 50)) AS i_class#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#8, 50)) AS i_category#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_category#8, i_class#7], [ws_net_paid#2, i_category#8, i_class#7] - -(14) CometExpand -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] -(15) ColumnarToRow [codegen id : 1] -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] +(15) CometExpand +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] -(16) HashAggregate [codegen id : 1] -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(16) CometHashAggregate +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#12] -Results [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -(17) Exchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(17) CometExchange +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(18) HashAggregate [codegen id : 2] -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(18) CometColumnarToRow [codegen id : 1] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] + +(19) HashAggregate [codegen id : 1] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#14] -Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS total_sum#15, i_category#9, i_class#10, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS lochierarchy#16, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS _w0#17, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS _w1#18, CASE WHEN (cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint) = 0) THEN i_category#9 END AS _w2#19] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#15] +Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS total_sum#16, i_category#11, i_class#12, (cast((shiftright(spark_grouping_id#13, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint)) AS lochierarchy#17, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS _w0#18, (cast((shiftright(spark_grouping_id#13, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint)) AS _w1#19, CASE WHEN (cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint) = 0) THEN i_category#11 END AS _w2#20] + +(20) CometColumnarExchange +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: hashpartitioning(_w1#19, _w2#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(19) Exchange -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(21) CometSort +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20], [_w1#19 ASC NULLS FIRST, _w2#20 ASC NULLS FIRST, _w0#18 DESC NULLS LAST] -(20) Sort [codegen id : 3] -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST], false, 0 +(22) CometColumnarToRow [codegen id : 2] +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] -(21) Window -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] +(23) Window +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: [rank(_w0#18) windowspecdefinition(_w1#19, _w2#20, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#21], [_w1#19, _w2#20], [_w0#18 DESC NULLS LAST] -(22) Project [codegen id : 4] -Output [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Input [8]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] +(24) Project [codegen id : 3] +Output [5]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] +Input [8]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20, rank_within_parent#21] -(23) TakeOrderedAndProject -Input [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] +(25) TakeOrderedAndProject +Input [5]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] +Arguments: 100, [lochierarchy#17 DESC NULLS LAST, CASE WHEN (lochierarchy#17 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#21 ASC NULLS FIRST], [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/extended.txt new file mode 100644 index 0000000000..6429c3e05f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/extended.txt @@ -0,0 +1,27 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ws_net_paid#1))#2,17,2) AS total_sum#3, i_category#4, i_class#5, (cast((shiftright(spark_grouping_id#6, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#6, 0) & 1) as tinyint)) AS lochierarchy#7, MakeDecimal(sum(UnscaledValue(ws_net_paid#1))#2,17,2) AS _w0#8, (cast((shiftright(spark_grouping_id#6, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#6, 0) & 1) as tinyint)) AS _w1#9, CASE WHEN (cast((shiftright(spark_grouping_id#6, 0) & 1) as tinyint) = 0) THEN i_category#4 END AS _w2#10)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 19 out of 23 eligible operators (82%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/simplified.txt index aa40577d65..2ed965eeea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/simplified.txt @@ -1,31 +1,31 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (3) - Sort [_w1,_w2,_w0] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] - CometProject [ws_net_paid,i_category,i_class] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] - CometProject [ws_item_sk,ws_net_paid] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ws_net_paid] [i_category,i_class,spark_grouping_id,sum] + CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] + CometProject [ws_net_paid,i_category,i_class] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/explain.txt index 3ff75a4a3b..e13b9a1a1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/explain.txt @@ -1,45 +1,75 @@ == Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin LeftAnti BuildRight (24) - :- * BroadcastHashJoin LeftAnti BuildRight (22) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometHashAggregate (19) - : +- ReusedExchange (18) - +- ReusedExchange (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* HashAggregate (51) ++- * CometColumnarToRow (50) + +- CometColumnarExchange (49) + +- * HashAggregate (48) + +- * Project (47) + +- * BroadcastHashJoin LeftAnti BuildRight (46) + :- * BroadcastHashJoin LeftAnti BuildRight (32) + : :- * CometColumnarToRow (18) + : : +- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (31) + : +- * CometColumnarToRow (30) + : +- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometFilter (20) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (19) + : : +- ReusedExchange (21) + : +- ReusedExchange (24) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : +- ReusedExchange (35) + +- ReusedExchange (38) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] @@ -62,93 +92,199 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(9) CometNativeScan: `spark_catalog`.`default`.`customer` +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#7, 20)) AS c_first_name#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#8, 30)) AS c_last_name#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(17) ColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(18) CometColumnarToRow [codegen id : 3] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] + +(19) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) + +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(18) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] +(23) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(19) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] +(24) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] + +(25) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight + +(26) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] + +(27) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(20) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] +(28) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(21) BroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=2] +(29) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] +Functions: [] -(22) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)] +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] + +(31) BroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] + +(32) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None -(23) ReusedExchange [Reuses operator id: 21] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(24) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] +(35) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight + +(37) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] + +(38) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] + +(39) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] + +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(42) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(44) CometColumnarToRow [codegen id : 2] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] + +(45) BroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] + +(46) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None -(25) Project [codegen id : 3] +(47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(26) HashAggregate [codegen id : 3] +(48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [1]: [count#16] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] + +(49) CometColumnarExchange +Input [1]: [count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(27) Exchange -Input [1]: [count#16] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [count#26] -(28) HashAggregate [codegen id : 4] -Input [1]: [count#16] +(51) HashAggregate [codegen id : 4] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [1]: [count(1)#17 AS count(1)#18] +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/extended.txt new file mode 100644 index 0000000000..f659e21953 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/extended.txt @@ -0,0 +1,65 @@ +HashAggregate ++- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 52 out of 59 eligible operators (88%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/simplified.txt index 8612445652..0fe68e3efa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/simplified.txt @@ -1,36 +1,61 @@ WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - ColumnarToRow + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #2 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #3 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - InputAdapter - ReusedExchange [c_last_name,c_first_name,d_date] #5 + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/explain.txt index 1917778c34..25b4c305f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/explain.txt @@ -6,7 +6,7 @@ : : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) : : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : : : : : :- * ColumnarToRow (25) +: : : : : : :- * CometColumnarToRow (25) : : : : : : : +- CometHashAggregate (24) : : : : : : : +- CometExchange (23) : : : : : : : +- CometHashAggregate (22) @@ -18,21 +18,21 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (16) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) -: : : : : : +- * ColumnarToRow (44) +: : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) : : : : : : +- CometExchange (42) : : : : : : +- CometHashAggregate (41) @@ -44,15 +44,15 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) -: : : : : +- * ColumnarToRow (65) +: : : : : +- * CometColumnarToRow (65) : : : : : +- CometHashAggregate (64) : : : : : +- CometExchange (63) : : : : : +- CometHashAggregate (62) @@ -64,15 +64,15 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (53) +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) -: : : : +- * ColumnarToRow (86) +: : : : +- * CometColumnarToRow (86) : : : : +- CometHashAggregate (85) : : : : +- CometExchange (84) : : : : +- CometHashAggregate (83) @@ -84,15 +84,15 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (68) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (74) +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) -: : : +- * ColumnarToRow (107) +: : : +- * CometColumnarToRow (107) : : : +- CometHashAggregate (106) : : : +- CometExchange (105) : : : +- CometHashAggregate (104) @@ -104,15 +104,15 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (89) +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (95) +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) -: : +- * ColumnarToRow (128) +: : +- * CometColumnarToRow (128) : : +- CometHashAggregate (127) : : +- CometExchange (126) : : +- CometHashAggregate (125) @@ -124,15 +124,15 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (110) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (116) +: : : +- CometNativeScan parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) -: +- * ColumnarToRow (149) +: +- * CometColumnarToRow (149) : +- CometHashAggregate (148) : +- CometExchange (147) : +- CometHashAggregate (146) @@ -144,15 +144,15 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (131) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (137) +: : +- CometNativeScan parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) - +- * ColumnarToRow (170) + +- * CometColumnarToRow (170) +- CometHashAggregate (169) +- CometExchange (168) +- CometHashAggregate (167) @@ -164,18 +164,21 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (152) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (158) + : +- CometNativeScan parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] @@ -185,9 +188,12 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Arguments: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] @@ -210,9 +216,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Arguments: [t_time_sk#8, t_hour#9, t_minute#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] @@ -235,9 +244,12 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometNativeScan: `spark_catalog`.`default`.`store` +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] -Arguments: [s_store_sk#11, s_store_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [s_store_sk#11, s_store_name#12] @@ -273,12 +285,15 @@ Input [1]: [count#13] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 8] +(25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(26) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (27) CometFilter Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] @@ -300,9 +315,12 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Arguments: [t_time_sk#20, t_hour#21, t_minute#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (33) CometFilter Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] @@ -350,7 +368,7 @@ Input [1]: [count#24] Keys: [] Functions [1]: [count(1)] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [h9_to_9_30#25] (45) BroadcastExchange @@ -361,9 +379,12 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(47) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (48) CometFilter Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] @@ -385,9 +406,12 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(53) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (54) CometFilter Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] @@ -435,7 +459,7 @@ Input [1]: [count#35] Keys: [] Functions [1]: [count(1)] -(65) ColumnarToRow [codegen id : 2] +(65) CometColumnarToRow [codegen id : 2] Input [1]: [h9_30_to_10#36] (66) BroadcastExchange @@ -446,9 +470,12 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(68) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (69) CometFilter Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] @@ -470,9 +497,12 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(74) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Arguments: [t_time_sk#42, t_hour#43, t_minute#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (75) CometFilter Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] @@ -520,7 +550,7 @@ Input [1]: [count#46] Keys: [] Functions [1]: [count(1)] -(86) ColumnarToRow [codegen id : 3] +(86) CometColumnarToRow [codegen id : 3] Input [1]: [h10_to_10_30#47] (87) BroadcastExchange @@ -531,9 +561,12 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(89) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (90) CometFilter Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] @@ -555,9 +588,12 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(95) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Arguments: [t_time_sk#53, t_hour#54, t_minute#55] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (96) CometFilter Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] @@ -605,7 +641,7 @@ Input [1]: [count#57] Keys: [] Functions [1]: [count(1)] -(107) ColumnarToRow [codegen id : 4] +(107) CometColumnarToRow [codegen id : 4] Input [1]: [h10_30_to_11#58] (108) BroadcastExchange @@ -616,9 +652,12 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(110) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (111) CometFilter Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] @@ -640,9 +679,12 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(116) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Arguments: [t_time_sk#64, t_hour#65, t_minute#66] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (117) CometFilter Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] @@ -690,7 +732,7 @@ Input [1]: [count#68] Keys: [] Functions [1]: [count(1)] -(128) ColumnarToRow [codegen id : 5] +(128) CometColumnarToRow [codegen id : 5] Input [1]: [h11_to_11_30#69] (129) BroadcastExchange @@ -701,9 +743,12 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(131) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (132) CometFilter Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] @@ -725,9 +770,12 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(137) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Arguments: [t_time_sk#75, t_hour#76, t_minute#77] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (138) CometFilter Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] @@ -775,7 +823,7 @@ Input [1]: [count#79] Keys: [] Functions [1]: [count(1)] -(149) ColumnarToRow [codegen id : 6] +(149) CometColumnarToRow [codegen id : 6] Input [1]: [h11_30_to_12#80] (150) BroadcastExchange @@ -786,9 +834,12 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(152) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (153) CometFilter Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] @@ -810,9 +861,12 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(158) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Arguments: [t_time_sk#86, t_hour#87, t_minute#88] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (159) CometFilter Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] @@ -860,7 +914,7 @@ Input [1]: [count#90] Keys: [] Functions [1]: [count(1)] -(170) ColumnarToRow [codegen id : 7] +(170) CometColumnarToRow [codegen id : 7] Input [1]: [h12_to_12_30#91] (171) BroadcastExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/extended.txt new file mode 100644 index 0000000000..779053cb5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/extended.txt @@ -0,0 +1,216 @@ +BroadcastNestedLoopJoin +:- BroadcastNestedLoopJoin +: :- BroadcastNestedLoopJoin +: : :- BroadcastNestedLoopJoin +: : : :- BroadcastNestedLoopJoin +: : : : :- BroadcastNestedLoopJoin +: : : : : :- BroadcastNestedLoopJoin +: : : : : : :- CometColumnarToRow +: : : : : : : +- CometHashAggregate +: : : : : : : +- CometExchange +: : : : : : : +- CometHashAggregate +: : : : : : : +- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : :- CometProject +: : : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : : :- CometProject +: : : : : : : : : : +- CometFilter +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : : : +- CometBroadcastExchange +: : : : : : : : : +- CometProject +: : : : : : : : : +- CometFilter +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : : : +- CometBroadcastExchange +: : : : : : : : +- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : : : +- BroadcastExchange +: : : : : : +- CometColumnarToRow +: : : : : : +- CometHashAggregate +: : : : : : +- CometExchange +: : : : : : +- CometHashAggregate +: : : : : : +- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : :- CometProject +: : : : : : : : : +- CometFilter +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : : +- CometBroadcastExchange +: : : : : : : : +- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : : +- BroadcastExchange +: : : : : +- CometColumnarToRow +: : : : : +- CometHashAggregate +: : : : : +- CometExchange +: : : : : +- CometHashAggregate +: : : : : +- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : +- BroadcastExchange +: : : : +- CometColumnarToRow +: : : : +- CometHashAggregate +: : : : +- CometExchange +: : : : +- CometHashAggregate +: : : : +- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- BroadcastExchange +: : : +- CometColumnarToRow +: : : +- CometHashAggregate +: : : +- CometExchange +: : : +- CometHashAggregate +: : : +- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.store +: : +- BroadcastExchange +: : +- CometColumnarToRow +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometHashAggregate +: +- CometExchange +: +- CometHashAggregate +: +- CometProject +: +- CometBroadcastHashJoin +: :- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.time_dim +: +- CometBroadcastExchange +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.store ++- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 192 out of 206 eligible operators (93%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/simplified.txt index 5acfeba131..cef8e0e760 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/simplified.txt @@ -6,9 +6,9 @@ WholeStageCodegen (8) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h8_30_to_9,count,count(1)] + CometHashAggregate [count] [h8_30_to_9,count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -19,25 +19,25 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h9_to_9_30,count,count(1)] + CometHashAggregate [count] [h9_to_9_30,count(1)] CometExchange #6 CometHashAggregate [count] CometProject @@ -48,19 +48,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 WholeStageCodegen (2) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h9_30_to_10,count,count(1)] + CometHashAggregate [count] [h9_30_to_10,count(1)] CometExchange #9 CometHashAggregate [count] CometProject @@ -71,19 +71,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h10_to_10_30,count,count(1)] + CometHashAggregate [count] [h10_to_10_30,count(1)] CometExchange #12 CometHashAggregate [count] CometProject @@ -94,19 +94,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 WholeStageCodegen (4) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h10_30_to_11,count,count(1)] + CometHashAggregate [count] [h10_30_to_11,count(1)] CometExchange #15 CometHashAggregate [count] CometProject @@ -117,19 +117,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h11_to_11_30,count,count(1)] + CometHashAggregate [count] [h11_to_11_30,count(1)] CometExchange #18 CometHashAggregate [count] CometProject @@ -140,19 +140,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 WholeStageCodegen (6) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h11_30_to_12,count,count(1)] + CometHashAggregate [count] [h11_30_to_12,count(1)] CometExchange #21 CometHashAggregate [count] CometProject @@ -163,19 +163,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 WholeStageCodegen (7) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h12_to_12_30,count,count(1)] + CometHashAggregate [count] [h12_to_12_30,count(1)] CometExchange #24 CometHashAggregate [count] CometProject @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/explain.txt index 75ee453e4b..026597e86c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/explain.txt @@ -1,151 +1,171 @@ == Physical Plan == -TakeOrderedAndProject (28) -+- * Project (27) - +- * Filter (26) - +- Window (25) - +- * Sort (24) - +- Exchange (23) - +- * HashAggregate (22) - +- Exchange (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (30) ++- * Project (29) + +- * Filter (28) + +- Window (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(4) CometFilter -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Right output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_item_sk#1], [ss_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(11) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] - -(12) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(15) CometFilter -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) - -(16) CometBroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(17) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Right output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [ss_store_sk#6], [s_store_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14], [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] - -(20) HashAggregate [codegen id : 1] -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum#15] -Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] - -(21) Exchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 2] -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#17] -Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS _w0#19] - -(23) Exchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(24) Sort [codegen id : 3] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST], false, 0 - -(25) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] - -(26) Filter [codegen id : 4] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] -Condition : CASE WHEN NOT (avg_monthly_sales#20 = 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END - -(27) Project [codegen id : 4] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] - -(28) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Books ,Electronics ,Sports ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (computers ,stereo ,football )) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Men ,Jewelry ,Women ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) + +(3) CometProject +Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) AS i_class#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) AS i_category#7] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Condition : (isnotnull(ss_item_sk#8) AND isnotnull(ss_store_sk#9)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] + +(7) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7] +Right output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight + +(8) CometProject +Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) + +(11) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] + +(12) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] + +(13) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight + +(14) CometProject +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) + +(17) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] + +(18) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight + +(19) CometProject +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] + +(20) CometHashAggregate +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] + +(21) CometExchange +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] + +(23) HashAggregate [codegen id : 1] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#10))#19] +Results [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#19,17,2) AS _w0#21] + +(24) CometColumnarExchange +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(25) CometSort +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] + +(26) CometColumnarToRow [codegen id : 2] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] + +(27) Window +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] + +(28) Filter [codegen id : 3] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21, avg_monthly_sales#22] +Condition : CASE WHEN NOT (avg_monthly_sales#22 = 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) END + +(29) Project [codegen id : 3] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21, avg_monthly_sales#22] + +(30) TakeOrderedAndProject +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [(sum_sales#20 - avg_monthly_sales#22) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/extended.txt new file mode 100644 index 0000000000..c2eb0c2841 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/extended.txt @@ -0,0 +1,32 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#1, i_class#2, i_brand#3, s_store_name#4, s_company_name#5, d_moy#6, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#8,17,2) AS sum_sales#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#8,17,2) AS _w0#10)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 23 out of 28 eligible operators (82%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/simplified.txt index cf2bafd365..c5f571e999 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/simplified.txt @@ -1,36 +1,36 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy] - WholeStageCodegen (4) + WholeStageCodegen (3) Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales] Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (3) - Sort [i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] + CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] CometFilter [i_item_sk,i_brand,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_moy] #4 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/explain.txt index e17d267111..da194f2825 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/explain.txt @@ -1,41 +1,47 @@ == Physical Plan == -* ColumnarToRow (4) -+- CometProject (3) +* Project (4) ++- * CometColumnarToRow (3) +- CometFilter (2) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (1) + +- CometNativeScan parquet spark_catalog.default.reason (1) -(1) CometNativeScan: `spark_catalog`.`default`.`reason` +(1) CometNativeScan parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] -Arguments: [r_reason_sk#1] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)] +ReadSchema: struct (2) CometFilter Input [1]: [r_reason_sk#1] Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) -(3) CometProject +(3) CometColumnarToRow [codegen id : 1] Input [1]: [r_reason_sk#1] -Arguments: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6], [CASE WHEN (Subquery scalar-subquery#7, [id=#8].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_net_paid) END AS bucket1#2, CASE WHEN (Subquery scalar-subquery#9, [id=#10].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_net_paid) END AS bucket2#3, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket3#4, CASE WHEN (Subquery scalar-subquery#13, [id=#14].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_net_paid) END AS bucket4#5, CASE WHEN (Subquery scalar-subquery#15, [id=#16].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_net_paid) END AS bucket5#6] -(4) ColumnarToRow [codegen id : 1] -Input [5]: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6] +(4) Project [codegen id : 1] +Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] +Input [1]: [r_reason_sk#1] ===== Subqueries ===== -Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#7, [id=#8] -* Project (12) -+- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * ColumnarToRow (8) +Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] +* CometColumnarToRow (12) ++- CometProject (11) + +- CometHashAggregate (10) + +- CometExchange (9) + +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) + +- CometNativeScan parquet spark_catalog.default.store_sales (5) -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)] +ReadSchema: struct (6) CometFilter Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] @@ -45,241 +51,233 @@ Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quant Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Arguments: [ss_ext_discount_amt#18, ss_net_paid#19], [ss_ext_discount_amt#18, ss_net_paid#19] -(8) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] - -(9) HashAggregate [codegen id : 1] +(8) CometHashAggregate Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [5]: [count#21, sum#22, count#23, sum#24, count#25] -Results [5]: [count#26, sum#27, count#28, sum#29, count#30] -(10) Exchange -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] +(9) CometExchange +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(11) HashAggregate [codegen id : 2] -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] +(10) CometHashAggregate +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [3]: [count(1)#31, avg(UnscaledValue(ss_ext_discount_amt#18))#32, avg(UnscaledValue(ss_net_paid#19))#33] -Results [3]: [count(1)#31 AS count(1)#34, cast((avg(UnscaledValue(ss_ext_discount_amt#18))#32 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#35, cast((avg(UnscaledValue(ss_net_paid#19))#33 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#36] -(12) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#34, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#35, avg(ss_net_paid), avg(ss_net_paid)#36) AS mergedValue#37] -Input [3]: [count(1)#34, avg(ss_ext_discount_amt)#35, avg(ss_net_paid)#36] +(11) CometProject +Input [3]: [count(1)#26, avg(ss_ext_discount_amt)#27, avg(ss_net_paid)#28] +Arguments: [mergedValue#29], [named_struct(count(1), count(1)#26, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#27, avg(ss_net_paid), avg(ss_net_paid)#28) AS mergedValue#29] + +(12) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#29] -Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] +Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] -Subquery:3 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] +Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] -Subquery:4 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* Project (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * ColumnarToRow (16) +Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] +* CometColumnarToRow (20) ++- CometProject (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) + +- CometNativeScan parquet spark_catalog.default.store_sales (13) -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Arguments: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] +(13) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] +ReadSchema: struct (14) CometFilter -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Condition : ((isnotnull(ss_quantity#38) AND (ss_quantity#38 >= 21)) AND (ss_quantity#38 <= 40)) +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Condition : ((isnotnull(ss_quantity#30) AND (ss_quantity#30 >= 21)) AND (ss_quantity#30 <= 40)) (15) CometProject -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Arguments: [ss_ext_discount_amt#39, ss_net_paid#40], [ss_ext_discount_amt#39, ss_net_paid#40] +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Arguments: [ss_ext_discount_amt#31, ss_net_paid#32], [ss_ext_discount_amt#31, ss_net_paid#32] -(16) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] - -(17) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] +(16) CometHashAggregate +Input [2]: [ss_ext_discount_amt#31, ss_net_paid#32] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#39)), partial_avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [5]: [count#42, sum#43, count#44, sum#45, count#46] -Results [5]: [count#47, sum#48, count#49, sum#50, count#51] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] -(18) Exchange -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(17) CometExchange +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(19) HashAggregate [codegen id : 2] -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +(18) CometHashAggregate +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#39)), avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [3]: [count(1)#52, avg(UnscaledValue(ss_ext_discount_amt#39))#53, avg(UnscaledValue(ss_net_paid#40))#54] -Results [3]: [count(1)#52 AS count(1)#55, cast((avg(UnscaledValue(ss_ext_discount_amt#39))#53 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#56, cast((avg(UnscaledValue(ss_net_paid#40))#54 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#57] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(UnscaledValue(ss_net_paid#32))] + +(19) CometProject +Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] +Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] -(20) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#55, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#56, avg(ss_net_paid), avg(ss_net_paid)#57) AS mergedValue#58] -Input [3]: [count(1)#55, avg(ss_ext_discount_amt)#56, avg(ss_net_paid)#57] +(20) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#42] -Subquery:5 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] -Subquery:6 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] -Subquery:7 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* Project (28) -+- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) +Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] +* CometColumnarToRow (28) ++- CometProject (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (21) + +- CometNativeScan parquet spark_catalog.default.store_sales (21) -(21) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Arguments: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] +(21) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] +ReadSchema: struct (22) CometFilter -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Condition : ((isnotnull(ss_quantity#59) AND (ss_quantity#59 >= 41)) AND (ss_quantity#59 <= 60)) +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Condition : ((isnotnull(ss_quantity#43) AND (ss_quantity#43 >= 41)) AND (ss_quantity#43 <= 60)) (23) CometProject -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Arguments: [ss_ext_discount_amt#60, ss_net_paid#61], [ss_ext_discount_amt#60, ss_net_paid#61] +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Arguments: [ss_ext_discount_amt#44, ss_net_paid#45], [ss_ext_discount_amt#44, ss_net_paid#45] -(24) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] - -(25) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] +(24) CometHashAggregate +Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#60)), partial_avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [5]: [count#63, sum#64, count#65, sum#66, count#67] -Results [5]: [count#68, sum#69, count#70, sum#71, count#72] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] -(26) Exchange -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +(25) CometExchange +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(27) HashAggregate [codegen id : 2] -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] +(26) CometHashAggregate +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#60)), avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [3]: [count(1)#73, avg(UnscaledValue(ss_ext_discount_amt#60))#74, avg(UnscaledValue(ss_net_paid#61))#75] -Results [3]: [count(1)#73 AS count(1)#76, cast((avg(UnscaledValue(ss_ext_discount_amt#60))#74 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#77, cast((avg(UnscaledValue(ss_net_paid#61))#75 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#78] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(UnscaledValue(ss_net_paid#45))] + +(27) CometProject +Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] +Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] -(28) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#76, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#77, avg(ss_net_paid), avg(ss_net_paid)#78) AS mergedValue#79] -Input [3]: [count(1)#76, avg(ss_ext_discount_amt)#77, avg(ss_net_paid)#78] +(28) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#55] -Subquery:8 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] -Subquery:9 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] -Subquery:10 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#13, [id=#14] -* Project (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * ColumnarToRow (32) +Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (36) ++- CometProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (29) + +- CometNativeScan parquet spark_catalog.default.store_sales (29) -(29) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Arguments: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] +(29) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] +ReadSchema: struct (30) CometFilter -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Condition : ((isnotnull(ss_quantity#80) AND (ss_quantity#80 >= 61)) AND (ss_quantity#80 <= 80)) +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Condition : ((isnotnull(ss_quantity#56) AND (ss_quantity#56 >= 61)) AND (ss_quantity#56 <= 80)) (31) CometProject -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Arguments: [ss_ext_discount_amt#81, ss_net_paid#82], [ss_ext_discount_amt#81, ss_net_paid#82] +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Arguments: [ss_ext_discount_amt#57, ss_net_paid#58], [ss_ext_discount_amt#57, ss_net_paid#58] -(32) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] - -(33) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] +(32) CometHashAggregate +Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#81)), partial_avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [5]: [count#84, sum#85, count#86, sum#87, count#88] -Results [5]: [count#89, sum#90, count#91, sum#92, count#93] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] -(34) Exchange -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(33) CometExchange +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(35) HashAggregate [codegen id : 2] -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] +(34) CometHashAggregate +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#81)), avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [3]: [count(1)#94, avg(UnscaledValue(ss_ext_discount_amt#81))#95, avg(UnscaledValue(ss_net_paid#82))#96] -Results [3]: [count(1)#94 AS count(1)#97, cast((avg(UnscaledValue(ss_ext_discount_amt#81))#95 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#98, cast((avg(UnscaledValue(ss_net_paid#82))#96 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#99] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(UnscaledValue(ss_net_paid#58))] + +(35) CometProject +Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] +Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] -(36) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#97, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#98, avg(ss_net_paid), avg(ss_net_paid)#99) AS mergedValue#100] -Input [3]: [count(1)#97, avg(ss_ext_discount_amt)#98, avg(ss_net_paid)#99] +(36) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#68] -Subquery:11 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] +Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:12 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] +Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:13 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#15, [id=#16] -* Project (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * ColumnarToRow (40) +Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] +* CometColumnarToRow (44) ++- CometProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (37) + +- CometNativeScan parquet spark_catalog.default.store_sales (37) -(37) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Arguments: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] +(37) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] +ReadSchema: struct (38) CometFilter -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Condition : ((isnotnull(ss_quantity#101) AND (ss_quantity#101 >= 81)) AND (ss_quantity#101 <= 100)) +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Condition : ((isnotnull(ss_quantity#69) AND (ss_quantity#69 >= 81)) AND (ss_quantity#69 <= 100)) (39) CometProject -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Arguments: [ss_ext_discount_amt#102, ss_net_paid#103], [ss_ext_discount_amt#102, ss_net_paid#103] +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Arguments: [ss_ext_discount_amt#70, ss_net_paid#71], [ss_ext_discount_amt#70, ss_net_paid#71] -(40) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] - -(41) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] +(40) CometHashAggregate +Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#102)), partial_avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [5]: [count#105, sum#106, count#107, sum#108, count#109] -Results [5]: [count#110, sum#111, count#112, sum#113, count#114] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] -(42) Exchange -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(41) CometExchange +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(43) HashAggregate [codegen id : 2] -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] +(42) CometHashAggregate +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#102)), avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [3]: [count(1)#115, avg(UnscaledValue(ss_ext_discount_amt#102))#116, avg(UnscaledValue(ss_net_paid#103))#117] -Results [3]: [count(1)#115 AS count(1)#118, cast((avg(UnscaledValue(ss_ext_discount_amt#102))#116 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#119, cast((avg(UnscaledValue(ss_net_paid#103))#117 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#120] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(UnscaledValue(ss_net_paid#71))] + +(43) CometProject +Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] +Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] -(44) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#118, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#119, avg(ss_net_paid), avg(ss_net_paid)#120) AS mergedValue#121] -Input [3]: [count(1)#118, avg(ss_ext_discount_amt)#119, avg(ss_net_paid)#120] +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#81] -Subquery:14 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] +Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] -Subquery:15 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] +Subquery:15 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/extended.txt new file mode 100644 index 0000000000..53ba3252cf --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/extended.txt @@ -0,0 +1,61 @@ + Project [COMET: ] +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: +- ReusedSubquery ++- CometColumnarToRow + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/simplified.txt index 919249c87f..9593d6ebd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/simplified.txt @@ -1,81 +1,71 @@ WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [bucket1,bucket2,bucket3,bucket4,bucket5] - Subquery #1 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #1 - ReusedSubquery [mergedValue] #1 - Subquery #2 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #2 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #2 - ReusedSubquery [mergedValue] #2 - Subquery #3 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #3 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #3 - ReusedSubquery [mergedValue] #3 - Subquery #4 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #4 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #4 - ReusedSubquery [mergedValue] #4 - Subquery #5 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #5 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #5 - ReusedSubquery [mergedValue] #5 + Project + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #1 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #1 + ReusedSubquery [mergedValue] #1 + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #2 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #2 + ReusedSubquery [mergedValue] #2 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #3 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #3 + ReusedSubquery [mergedValue] #3 + Subquery #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #4 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #4 + ReusedSubquery [mergedValue] #4 + Subquery #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #5 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #5 + ReusedSubquery [mergedValue] #5 + CometColumnarToRow + InputAdapter CometFilter [r_reason_sk] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/explain.txt index 6390d2d5eb..73b54f439b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * Project (47) +- * BroadcastNestedLoopJoin Inner BuildRight (46) - :- * ColumnarToRow (25) + :- * CometColumnarToRow (25) : +- CometHashAggregate (24) : +- CometExchange (23) : +- CometHashAggregate (22) @@ -13,21 +13,21 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (16) + : +- CometNativeScan parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) - +- * ColumnarToRow (44) + +- * CometColumnarToRow (44) +- CometHashAggregate (43) +- CometExchange (42) +- CometHashAggregate (41) @@ -39,18 +39,21 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) + : +- CometNativeScan parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] @@ -60,9 +63,12 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)] +ReadSchema: struct (5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] @@ -85,9 +91,12 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] -Arguments: [t_time_sk#7, t_hour#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)] +ReadSchema: struct (11) CometFilter Input [2]: [t_time_sk#7, t_hour#8] @@ -110,9 +119,12 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometNativeScan: `spark_catalog`.`default`.`web_page` +(16) CometNativeScan parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] -Arguments: [wp_web_page_sk#9, wp_char_count#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [wp_web_page_sk#9, wp_char_count#10] @@ -148,12 +160,15 @@ Input [1]: [count#11] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 2] +(25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(26) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] +ReadSchema: struct (27) CometFilter Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] @@ -175,9 +190,12 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(32) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] -Arguments: [t_time_sk#18, t_hour#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)] +ReadSchema: struct (33) CometFilter Input [2]: [t_time_sk#18, t_hour#19] @@ -225,7 +243,7 @@ Input [1]: [count#21] Keys: [] Functions [1]: [count(1)] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [pmc#22] (45) BroadcastExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/extended.txt new file mode 100644 index 0000000000..aa4016076b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/extended.txt @@ -0,0 +1,55 @@ +Project ++- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.time_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 48 out of 51 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/simplified.txt index a746bdf6f4..a13072cd91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (2) Project [amc,pmc] BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [amc,count,count(1)] + CometHashAggregate [count] [amc,count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -14,25 +14,25 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk,wp_char_count] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [pmc,count,count(1)] + CometHashAggregate [count] [pmc,count(1)] CometExchange #6 CometHashAggregate [count] CometProject @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/explain.txt index ecd5609ab4..dee1681304 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/explain.txt @@ -1,223 +1,253 @@ == Physical Plan == -* ColumnarToRow (42) -+- CometSort (41) - +- CometColumnarExchange (40) - +- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (13) - : : : : +- CometBroadcastHashJoin (12) - : : : : :- CometProject (7) - : : : : : +- CometBroadcastHashJoin (6) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (1) - : : : : : +- CometBroadcastExchange (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (3) - : : : : +- CometBroadcastExchange (11) - : : : : +- CometProject (10) - : : : : +- CometFilter (9) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : +- CometBroadcastExchange (16) - : : : +- CometFilter (15) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (14) - : : +- CometBroadcastExchange (22) - : : +- CometProject (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (19) - : +- CometBroadcastExchange (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (25) - +- CometBroadcastExchange (33) - +- CometProject (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`call_center` +* CometColumnarToRow (44) ++- CometSort (43) + +- CometColumnarExchange (42) + +- * HashAggregate (41) + +- * CometColumnarToRow (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometProject (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometFilter (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (4) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometNativeScan parquet spark_catalog.default.customer (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.household_demographics (32) + + +(1) CometNativeScan parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Arguments: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Condition : isnotnull(cc_call_center_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(4) CometFilter -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Right output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_sk#1], [cr_call_center_sk#6], Inner, BuildRight - -(7) CometProject -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(12) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [cr_returned_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(15) CometFilter -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) - -(16) CometBroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(17) CometBroadcastHashJoin -Left output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Right output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cr_returning_customer_sk#5], [c_customer_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16, ca_gmt_offset#17] - -(20) CometFilter -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) - -(21) CometProject -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(22) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(23) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Right output [1]: [ca_address_sk#16] -Arguments: [c_current_addr_sk#15], [ca_address_sk#16], Inner, BuildRight - -(24) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] - -(25) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(26) CometFilter -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) - -(27) CometBroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(28) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [c_current_cdemo_sk#13], [cd_demo_sk#18], Inner, BuildRight - -(29) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] - -(30) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21, hd_buy_potential#22] - -(31) CometFilter -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) - -(32) CometProject -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21], [hd_demo_sk#21] - -(33) CometBroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: [hd_demo_sk#21] - -(34) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Right output [1]: [hd_demo_sk#21] -Arguments: [c_current_hdemo_sk#14], [hd_demo_sk#21], Inner, BuildRight - -(35) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] - -(36) ColumnarToRow [codegen id : 1] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] - -(37) HashAggregate [codegen id : 1] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum#23] -Results [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] - -(38) Exchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(39) HashAggregate [codegen id : 2] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#7))#25] -Results [4]: [cc_call_center_id#2 AS Call_Center#26, cc_name#3 AS Call_Center_Name#27, cc_manager#4 AS Manager#28, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#25,17,2) AS Returns_Loss#29] - -(40) CometColumnarExchange -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: rangepartitioning(Returns_Loss#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(41) CometSort -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29], [Returns_Loss#29 DESC NULLS LAST] - -(42) ColumnarToRow [codegen id : 3] -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] +(3) CometProject +Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(cc_call_center_id#2, 16)) AS cc_call_center_id#5, cc_name#3, cc_manager#4] + +(4) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] +PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_call_center_sk#7) AND isnotnull(cr_returning_customer_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [4]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4] +Right output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight + +(8) CometProject +Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) + +(11) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] + +(13) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(14) CometProject +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] + +(15) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) + +(17) CometBroadcastExchange +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] + +(18) CometBroadcastHashJoin +Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight + +(19) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] + +(20) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) + +(22) CometProject +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] + +(23) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] + +(24) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight + +(25) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] + +(26) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(27) CometFilter +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Unknown )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) + +(28) CometProject +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) AS cd_marital_status#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) AS cd_education_status#23] + +(29) CometBroadcastExchange +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] + +(30) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight + +(31) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] + +(32) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : ((isnotnull(hd_buy_potential#25) AND StartsWith(static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#25, 15)), Unknown)) AND isnotnull(hd_demo_sk#24)) + +(34) CometProject +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] + +(35) CometBroadcastExchange +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] + +(36) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight + +(37) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] + +(38) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] + +(39) CometExchange +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometColumnarToRow [codegen id : 1] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] + +(41) HashAggregate [codegen id : 1] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#8))#27] +Results [4]: [cc_call_center_id#5 AS Call_Center#28, cc_name#3 AS Call_Center_Name#29, cc_manager#4 AS Manager#30, MakeDecimal(sum(UnscaledValue(cr_net_loss#8))#27,17,2) AS Returns_Loss#31] + +(42) CometColumnarExchange +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] +Arguments: rangepartitioning(Returns_Loss#31 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(43) CometSort +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] +Arguments: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31], [Returns_Loss#31 DESC NULLS LAST] + +(44) CometColumnarToRow [codegen id : 2] +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/extended.txt new file mode 100644 index 0000000000..2629f99199 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/extended.txt @@ -0,0 +1,46 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cc_call_center_id#1 AS Call_Center#2, cc_name#3 AS Call_Center_Name#4, cc_manager#5 AS Manager#6, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#8,17,2) AS Returns_Loss#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.household_demographics + +Comet accelerated 41 out of 42 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/simplified.txt index 652ca90404..80d05bbe08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/simplified.txt @@ -1,48 +1,48 @@ -WholeStageCodegen (3) - ColumnarToRow +WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] CometColumnarExchange [Returns_Loss] #1 - WholeStageCodegen (2) + WholeStageCodegen (1) HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum] - InputAdapter - Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - WholeStageCodegen (1) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] - CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 - CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] + CometColumnarToRow + InputAdapter + CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 + CometHashAggregate [cr_net_loss] [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 + CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/explain.txt index cdb99b45b3..24d3a3c5b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/explain.txt @@ -1,52 +1,54 @@ == Physical Plan == -* HashAggregate (35) -+- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- BroadcastExchange (23) - : +- * Filter (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (10) - : +- CometBroadcastExchange (15) - : +- CometProject (14) - : +- CometFilter (13) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - +- BroadcastExchange (30) - +- * ColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* HashAggregate (30) ++- * CometColumnarToRow (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] @@ -69,125 +71,105 @@ Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -(9) ColumnarToRow [codegen id : 4] -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] - -(10) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(9) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Arguments: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#8)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct -(11) CometFilter +(10) CometFilter Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Condition : isnotnull(ws_item_sk#6) -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] +ReadSchema: struct -(13) CometFilter +(12) CometFilter Input [2]: [d_date_sk#9, d_date#10] Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) -(14) CometProject +(13) CometProject Input [2]: [d_date_sk#9, d_date#10] Arguments: [d_date_sk#9], [d_date_sk#9] -(15) CometBroadcastExchange +(14) CometBroadcastExchange Input [1]: [d_date_sk#9] Arguments: [d_date_sk#9] -(16) CometBroadcastHashJoin +(15) CometBroadcastHashJoin Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Right output [1]: [d_date_sk#9] Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(17) CometProject +(16) CometProject Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] -(18) ColumnarToRow [codegen id : 1] -Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] - -(19) HashAggregate [codegen id : 1] +(17) CometHashAggregate Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] Keys [1]: [ws_item_sk#6] Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [ws_item_sk#6, sum#13, count#14] -(20) Exchange -Input [3]: [ws_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(18) CometExchange +Input [3]: [ws_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 2] -Input [3]: [ws_item_sk#6, sum#13, count#14] +(19) CometHashAggregate +Input [3]: [ws_item_sk#6, sum#11, count#12] Keys [1]: [ws_item_sk#6] Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(ws_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] - -(22) Filter [codegen id : 2] -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#16) -(23) BroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=2] +(20) CometFilter +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [ws_item_sk#6] -Join type: Inner -Join condition: (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#16) +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] -(25) Project [codegen id : 4] -Output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] +(22) CometBroadcastHashJoin +Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight -(26) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17, d_date#18] +(23) CometProject +Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] -(27) CometFilter -Input [2]: [d_date_sk#17, d_date#18] -Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-01-27)) AND (d_date#18 <= 2000-04-26)) AND isnotnull(d_date_sk#17)) +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#14] -(28) CometProject -Input [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17], [d_date_sk#17] +(25) CometBroadcastHashJoin +Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#14] +Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight -(29) ColumnarToRow [codegen id : 3] -Input [1]: [d_date_sk#17] +(26) CometProject +Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#14] +Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] -(30) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(31) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 4] -Output [1]: [ws_ext_discount_amt#2] -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#17] - -(33) HashAggregate [codegen id : 4] +(27) CometHashAggregate Input [1]: [ws_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#19] -Results [1]: [sum#20] -(34) Exchange -Input [1]: [sum#20] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [1]: [sum#15] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum#15] -(35) HashAggregate [codegen id : 5] -Input [1]: [sum#20] +(30) HashAggregate [codegen id : 1] +Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#21] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#21,17,2) AS Excess Discount Amount #22] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#16] +Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#16,17,2) AS Excess Discount Amount #17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/extended.txt new file mode 100644 index 0000000000..f93517a4c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/extended.txt @@ -0,0 +1,35 @@ + HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#1))#2,17,2) AS Excess Discount Amount #3)] ++- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 31 out of 32 eligible operators (96%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/simplified.txt index 6ba2c8ff43..d6af0449f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/simplified.txt @@ -1,47 +1,32 @@ -WholeStageCodegen (5) +WholeStageCodegen (1) HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [ws_ext_discount_amt] [sum,sum] - Project [ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_discount_amt,ws_sold_date_sk] - BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] - ColumnarToRow - InputAdapter - CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk] #2 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [(1.3 * avg(ws_ext_discount_amt))] - HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (1) - HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_ext_discount_amt] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [ws_ext_discount_amt] [sum] + CometProject [ws_ext_discount_amt] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk,(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 + CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] + CometExchange [ws_item_sk] #4 + CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] + CometProject [ws_item_sk,ws_ext_discount_amt] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/explain.txt index 58d4ecf05c..4b430e436e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * HashAggregate (21) - +- Exchange (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) +* CometColumnarToRow (22) ++- CometTakeOrderedAndProject (21) + +- CometHashAggregate (20) + +- CometExchange (19) + +- CometHashAggregate (18) +- CometProject (17) +- CometBroadcastHashJoin (16) :- CometProject (11) @@ -11,21 +11,23 @@ TakeOrderedAndProject (22) : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) + : +- CometNativeScan parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (12) + +- CometNativeScan parquet spark_catalog.default.reason (12) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +ReadSchema: struct (2) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] @@ -39,9 +41,12 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)] +ReadSchema: struct (6) CometFilter Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] @@ -68,13 +73,16 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometNativeScan: `spark_catalog`.`default`.`reason` +(12) CometNativeScan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] -Arguments: [r_reason_sk#12, r_reason_desc#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_desc), IsNotNull(r_reason_sk)] +ReadSchema: struct (13) CometFilter Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) +Condition : ((isnotnull(r_reason_desc#13) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(r_reason_desc#13, 100)) = reason 28 )) AND isnotnull(r_reason_sk#12)) (14) CometProject Input [2]: [r_reason_sk#12, r_reason_desc#13] @@ -93,28 +101,24 @@ Arguments: [sr_reason_sk#8], [r_reason_sk#12], Inner, BuildRight Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10, r_reason_sk#12] Arguments: [ss_customer_sk#2, act_sales#14], [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] -(18) ColumnarToRow [codegen id : 1] -Input [2]: [ss_customer_sk#2, act_sales#14] - -(19) HashAggregate [codegen id : 1] +(18) CometHashAggregate Input [2]: [ss_customer_sk#2, act_sales#14] Keys [1]: [ss_customer_sk#2] Functions [1]: [partial_sum(act_sales#14)] -Aggregate Attributes [2]: [sum#15, isEmpty#16] -Results [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -(20) Exchange -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(19) CometExchange +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(21) HashAggregate [codegen id : 2] -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] +(20) CometHashAggregate +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] Keys [1]: [ss_customer_sk#2] Functions [1]: [sum(act_sales#14)] -Aggregate Attributes [1]: [sum(act_sales#14)#19] -Results [2]: [ss_customer_sk#2, sum(act_sales#14)#19 AS sumsales#20] -(22) TakeOrderedAndProject -Input [2]: [ss_customer_sk#2, sumsales#20] -Arguments: 100, [sumsales#20 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#20] +(21) CometTakeOrderedAndProject +Input [2]: [ss_customer_sk#2, sumsales#17] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, 0, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] + +(22) CometColumnarToRow [codegen id : 1] +Input [2]: [ss_customer_sk#2, sumsales#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/extended.txt new file mode 100644 index 0000000000..7203a678b0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/extended.txt @@ -0,0 +1,24 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 21 out of 21 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/simplified.txt index 8d70da4080..c0b74010df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/simplified.txt @@ -1,26 +1,24 @@ -TakeOrderedAndProject [sumsales,ss_customer_sk] - WholeStageCodegen (2) - HashAggregate [ss_customer_sk,sum,isEmpty] [sum(act_sales),sumsales,sum,isEmpty] - InputAdapter - Exchange [ss_customer_sk] #1 - WholeStageCodegen (1) - HashAggregate [ss_customer_sk,act_sales] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometExchange [ss_item_sk,ss_ticket_number] #2 - CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometExchange [sr_item_sk,sr_ticket_number] #3 - CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [r_reason_sk] #4 - CometProject [r_reason_sk] - CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ss_customer_sk,sumsales] + CometHashAggregate [sum,isEmpty] [ss_customer_sk,sumsales,sum(act_sales)] + CometExchange [ss_customer_sk] #1 + CometHashAggregate [act_sales] [ss_customer_sk,sum,isEmpty] + CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometExchange [ss_item_sk,ss_ticket_number] #2 + CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometExchange [sr_item_sk,sr_ticket_number] #3 + CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [r_reason_sk] #4 + CometProject [r_reason_sk] + CometFilter [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/explain.txt index fde4e750e9..2c59d514be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/explain.txt @@ -1,49 +1,53 @@ == Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * HashAggregate (36) - +- * ColumnarToRow (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* HashAggregate (41) ++- * CometColumnarToRow (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometNativeScan parquet spark_catalog.default.web_site (29) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] @@ -61,9 +65,11 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] -Arguments: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +ReadSchema: struct (7) CometProject Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] @@ -86,9 +92,11 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(12) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] -Arguments: [wr_order_number#12, wr_returned_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +ReadSchema: struct (13) CometProject Input [2]: [wr_order_number#12, wr_returned_date_sk#13] @@ -107,9 +115,12 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct (18) CometFilter Input [2]: [d_date_sk#14, d_date#15] @@ -132,13 +143,16 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct (24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) +Condition : ((isnotnull(ca_state#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#17, 2)) = IL)) AND isnotnull(ca_address_sk#16)) (25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] @@ -157,13 +171,16 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometNativeScan: `spark_catalog`.`default`.`web_site` +(29) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] -Arguments: [web_site_sk#18, web_company_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_company_name), IsNotNull(web_site_sk)] +ReadSchema: struct (30) CometFilter Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) +Condition : ((isnotnull(web_company_name#19) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(web_company_name#19, 50)) = pri )) AND isnotnull(web_site_sk#18)) (31) CometProject Input [2]: [web_site_sk#18, web_company_name#19] @@ -182,38 +199,39 @@ Arguments: [ws_web_site_sk#3], [web_site_sk#18], Inner, BuildRight Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#18] Arguments: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(35) ColumnarToRow [codegen id : 1] -Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(36) HashAggregate [codegen id : 1] +(35) CometHashAggregate Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Keys [1]: [ws_order_number#5] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] + +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] (37) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#5, sum#22, sum#23] +Input [3]: [ws_order_number#5, sum#20, sum#21] Keys [1]: [ws_order_number#5] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23] +Results [3]: [ws_order_number#5, sum#20, sum#21] (38) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#5, sum#22, sum#23] +Input [3]: [ws_order_number#5, sum#20, sum#21] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] + +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(39) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarToRow [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] -(40) HashAggregate [codegen id : 2] -Input [3]: [sum#22, sum#23, count#25] +(41) HashAggregate [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#21,17,2) AS total net profit #28] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] +Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#23,17,2) AS total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/extended.txt new file mode 100644 index 0000000000..2768e8f638 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/extended.txt @@ -0,0 +1,43 @@ + HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(count(ws_order_number#1)#2 AS order count #3, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#4))#5,17,2) AS total shipping cost #6, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#8,17,2) AS total net profit #9)] ++- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 36 out of 39 eligible operators (92%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/simplified.txt index aad98bbfcf..49da76bc06 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/simplified.txt @@ -1,44 +1,45 @@ WholeStageCodegen (2) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow InputAdapter - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/explain.txt index 164424d188..52d66c157f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/explain.txt @@ -1,62 +1,66 @@ == Physical Plan == -* HashAggregate (53) -+- Exchange (52) - +- * HashAggregate (51) - +- * HashAggregate (50) - +- * HashAggregate (49) - +- * ColumnarToRow (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometSortMergeJoin (29) - : : : :- CometSortMergeJoin (15) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometProject (14) - : : : : +- CometSortMergeJoin (13) - : : : : :- CometSort (10) - : : : : : +- CometExchange (9) - : : : : : +- CometProject (8) - : : : : : +- CometFilter (7) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : : +- CometSort (12) - : : : : +- ReusedExchange (11) - : : : +- CometProject (28) - : : : +- CometSortMergeJoin (27) - : : : :- CometSort (20) - : : : : +- CometExchange (19) - : : : : +- CometProject (18) - : : : : +- CometFilter (17) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (16) - : : : +- CometProject (26) - : : : +- CometSortMergeJoin (25) - : : : :- CometSort (22) - : : : : +- ReusedExchange (21) - : : : +- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometBroadcastExchange (33) - : : +- CometProject (32) - : : +- CometFilter (31) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - : +- CometBroadcastExchange (39) - : +- CometProject (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - +- CometBroadcastExchange (45) - +- CometProject (44) - +- CometFilter (43) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* HashAggregate (54) ++- * CometColumnarToRow (53) + +- CometColumnarExchange (52) + +- * HashAggregate (51) + +- * HashAggregate (50) + +- * CometColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometBroadcastHashJoin (46) + :- CometProject (41) + : +- CometBroadcastHashJoin (40) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometSortMergeJoin (29) + : : : :- CometSortMergeJoin (15) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometProject (14) + : : : : +- CometSortMergeJoin (13) + : : : : :- CometSort (10) + : : : : : +- CometExchange (9) + : : : : : +- CometProject (8) + : : : : : +- CometFilter (7) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) + : : : : +- CometSort (12) + : : : : +- ReusedExchange (11) + : : : +- CometProject (28) + : : : +- CometSortMergeJoin (27) + : : : :- CometSort (20) + : : : : +- CometExchange (19) + : : : : +- CometProject (18) + : : : : +- CometFilter (17) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (16) + : : : +- CometProject (26) + : : : +- CometSortMergeJoin (25) + : : : :- CometSort (22) + : : : : +- ReusedExchange (21) + : : : +- CometSort (24) + : : : +- ReusedExchange (23) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : +- CometBroadcastExchange (39) + : +- CometProject (38) + : +- CometFilter (37) + : +- CometNativeScan parquet spark_catalog.default.customer_address (36) + +- CometBroadcastExchange (45) + +- CometProject (44) + +- CometFilter (43) + +- CometNativeScan parquet spark_catalog.default.web_site (42) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] @@ -74,9 +78,12 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Arguments: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] +ReadSchema: struct (7) CometFilter Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] @@ -115,9 +122,12 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(16) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] -Arguments: [wr_order_number#13, wr_returned_date_sk#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number)] +ReadSchema: struct (17) CometFilter Input [2]: [wr_order_number#13, wr_returned_date_sk#14] @@ -172,9 +182,12 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#20] -Arguments: [d_date_sk#19, d_date#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct (31) CometFilter Input [2]: [d_date_sk#19, d_date#20] @@ -197,13 +210,16 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(36) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#21, ca_state#22] -Arguments: [ca_address_sk#21, ca_state#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct (37) CometFilter Input [2]: [ca_address_sk#21, ca_state#22] -Condition : ((isnotnull(ca_state#22) AND (ca_state#22 = IL)) AND isnotnull(ca_address_sk#21)) +Condition : ((isnotnull(ca_state#22) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) = IL)) AND isnotnull(ca_address_sk#21)) (38) CometProject Input [2]: [ca_address_sk#21, ca_state#22] @@ -222,13 +238,16 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometNativeScan: `spark_catalog`.`default`.`web_site` +(42) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#23, web_company_name#24] -Arguments: [web_site_sk#23, web_company_name#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_company_name), IsNotNull(web_site_sk)] +ReadSchema: struct (43) CometFilter Input [2]: [web_site_sk#23, web_company_name#24] -Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri )) AND isnotnull(web_site_sk#23)) +Condition : ((isnotnull(web_company_name#24) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(web_company_name#24, 50)) = pri )) AND isnotnull(web_site_sk#23)) (44) CometProject Input [2]: [web_site_sk#23, web_company_name#24] @@ -247,38 +266,39 @@ Arguments: [ws_web_site_sk#3], [web_site_sk#23], Inner, BuildRight Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#23] Arguments: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(48) ColumnarToRow [codegen id : 1] -Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(49) HashAggregate [codegen id : 1] +(48) CometHashAggregate Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Keys [1]: [ws_order_number#4] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26] -Results [3]: [ws_order_number#4, sum#27, sum#28] + +(49) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#4, sum#25, sum#26] (50) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#4, sum#27, sum#28] +Input [3]: [ws_order_number#4, sum#25, sum#26] Keys [1]: [ws_order_number#4] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26] -Results [3]: [ws_order_number#4, sum#27, sum#28] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28] +Results [3]: [ws_order_number#4, sum#25, sum#26] (51) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#4, sum#27, sum#28] +Input [3]: [ws_order_number#4, sum#25, sum#26] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26, count(ws_order_number#4)#29] -Results [3]: [sum#27, sum#28, count#30] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] +Results [3]: [sum#25, sum#26, count#30] + +(52) CometColumnarExchange +Input [3]: [sum#25, sum#26, count#30] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(52) Exchange -Input [3]: [sum#27, sum#28, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(53) CometColumnarToRow [codegen id : 2] +Input [3]: [sum#25, sum#26, count#30] -(53) HashAggregate [codegen id : 2] -Input [3]: [sum#27, sum#28, count#30] +(54) HashAggregate [codegen id : 2] +Input [3]: [sum#25, sum#26, count#30] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26, count(ws_order_number#4)#29] -Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#25,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#26,17,2) AS total net profit #33] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] +Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#27,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#28,17,2) AS total net profit #33] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/extended.txt new file mode 100644 index 0000000000..074433c285 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/extended.txt @@ -0,0 +1,65 @@ + HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(count(ws_order_number#1)#2 AS order count #3, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#4))#5,17,2) AS total shipping cost #6, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#8,17,2) AS total net profit #9)] ++- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 58 out of 61 eligible operators (95%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/simplified.txt index 88077488d0..a7712046df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/simplified.txt @@ -1,57 +1,58 @@ WholeStageCodegen (2) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow InputAdapter - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometProject [ws_order_number] - CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometProject [wr_order_number] - CometSortMergeJoin [wr_order_number,ws_order_number] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] + CometProject [wr_order_number] + CometSortMergeJoin [wr_order_number,ws_order_number] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/explain.txt index 16424de5f3..db701aae34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (25) +* CometColumnarToRow (25) +- CometHashAggregate (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -11,24 +11,27 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) + : +- CometNativeScan parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometNativeScan: `spark_catalog`.`default`.`store` (16) + +- CometNativeScan parquet spark_catalog.default.store (16) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] @@ -38,9 +41,12 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)] +ReadSchema: struct (5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] @@ -63,9 +69,12 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Arguments: [t_time_sk#7, t_hour#8, t_minute#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] @@ -88,9 +97,12 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometNativeScan: `spark_catalog`.`default`.`store` +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] -Arguments: [s_store_sk#10, s_store_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [s_store_sk#10, s_store_name#11] @@ -126,6 +138,6 @@ Input [1]: [count#12] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [1]: [count(1)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/extended.txt new file mode 100644 index 0000000000..170d3b7879 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/simplified.txt index 1996d860c8..b40f41659f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/simplified.txt @@ -1,7 +1,7 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [count(1),count,count(1)] + CometHashAggregate [count] [count(1),count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/explain.txt index d0c03cb144..b5a88b6563 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* HashAggregate (25) -+- Exchange (24) - +- * HashAggregate (23) - +- * ColumnarToRow (22) +* CometColumnarToRow (25) ++- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) +- CometProject (21) +- CometSortMergeJoin (20) :- CometSort (11) @@ -11,28 +11,34 @@ : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : :- CometNativeScan parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (2) + : +- CometNativeScan parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) - :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (12) + :- CometNativeScan parquet spark_catalog.default.catalog_sales (12) +- ReusedExchange (13) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +ReadSchema: struct -(2) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(2) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (3) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -73,9 +79,12 @@ Functions: [] Input [2]: [customer_sk#6, item_sk#7] Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(12) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] -Arguments: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +ReadSchema: struct (13) ReusedExchange [Reuses operator id: 5] Output [1]: [d_date_sk#11] @@ -116,24 +125,20 @@ Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] -(22) ColumnarToRow [codegen id : 1] -Input [2]: [customer_sk#6, customer_sk#12] - -(23) HashAggregate [codegen id : 1] +(22) CometHashAggregate Input [2]: [customer_sk#6, customer_sk#12] Keys: [] Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum#14, sum#15, sum#16] -Results [3]: [sum#17, sum#18, sum#19] -(24) Exchange -Input [3]: [sum#17, sum#18, sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +(23) CometExchange +Input [3]: [sum#14, sum#15, sum#16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(25) HashAggregate [codegen id : 2] -Input [3]: [sum#17, sum#18, sum#19] +(24) CometHashAggregate +Input [3]: [sum#14, sum#15, sum#16] Keys: [] Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22] -Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20 AS store_only#23, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21 AS catalog_only#24, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22 AS store_and_catalog#25] + +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/extended.txt new file mode 100644 index 0000000000..07252f3fb5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/extended.txt @@ -0,0 +1,30 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometNativeScan parquet spark_catalog.default.catalog_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 27 out of 27 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/simplified.txt index 2f53fb71f6..18199546d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/simplified.txt @@ -1,29 +1,27 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [customer_sk,customer_sk] - CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] - CometExchange [ss_customer_sk,ss_item_sk] #2 - CometHashAggregate [ss_customer_sk,ss_item_sk] - CometProject [ss_item_sk,ss_customer_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometExchange [cs_bill_customer_sk,cs_item_sk] #4 - CometHashAggregate [cs_bill_customer_sk,cs_item_sk] - CometProject [cs_bill_customer_sk,cs_item_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 + CometHashAggregate [sum,sum,sum] [store_only,catalog_only,store_and_catalog,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] + CometExchange #1 + CometHashAggregate [customer_sk,customer_sk] [sum,sum,sum] + CometProject [customer_sk,customer_sk] + CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] + CometExchange [ss_customer_sk,ss_item_sk] #2 + CometHashAggregate [ss_customer_sk,ss_item_sk] + CometProject [ss_item_sk,ss_customer_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] + CometExchange [cs_bill_customer_sk,cs_item_sk] #4 + CometHashAggregate [cs_bill_customer_sk,cs_item_sk] + CometProject [cs_bill_customer_sk,cs_item_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/explain.txt index 3097726fc5..6826270748 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/explain.txt @@ -1,134 +1,151 @@ == Physical Plan == -* ColumnarToRow (25) -+- CometProject (24) - +- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * Sort (19) - +- Exchange (18) - +- * HashAggregate (17) - +- Exchange (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (27) ++- CometProject (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) ColumnarToRow [codegen id : 1] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(15) HashAggregate [codegen id : 1] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(16) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) HashAggregate [codegen id : 2] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] + +(19) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(18) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(20) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) Sort [codegen id : 3] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(22) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] -(21) Project [codegen id : 4] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(23) Project [codegen id : 3] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] -(22) CometColumnarExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +(24) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(23) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST] +(25) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] -(24) CometProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(26) CometProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] -(25) ColumnarToRow [codegen id : 5] -Input [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(27) CometColumnarToRow [codegen id : 4] +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/extended.txt new file mode 100644 index 0000000000..484d6b6a91 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/extended.txt @@ -0,0 +1,29 @@ +CometColumnarToRow ++- CometProject + +- CometSort + +- CometColumnarExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_desc#1, i_category#2, i_class#3, i_current_price#4, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#6,17,2) AS itemrevenue#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#6,17,2) AS _w0#8, i_item_id#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 21 out of 24 eligible operators (87%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/simplified.txt index 911dc29722..ca5771ad80 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/simplified.txt @@ -1,35 +1,35 @@ -WholeStageCodegen (5) - ColumnarToRow +WholeStageCodegen (4) + CometColumnarToRow InputAdapter CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (4) + WholeStageCodegen (3) Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] InputAdapter Window [_w0,i_class] - WholeStageCodegen (3) - Sort [i_class] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometColumnarExchange [i_class] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/explain.txt index cc6adee3c7..12e87af3b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/explain.txt @@ -1,41 +1,52 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.call_center (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] @@ -54,77 +65,104 @@ Arguments: [cs_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -(8) ReusedExchange [Reuses operator id: 5] +(8) CometNativeScan parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct -(9) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight +(9) CometFilter +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) (10) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_type#9, 30)) AS sm_type#10] -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [cc_call_center_sk#10, cc_name#11] +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] (12) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: [cs_call_center_sk#2], [cc_call_center_sk#10], Inner, BuildRight +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight (13) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#11, cc_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk)] +ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) ColumnarToRow [codegen id : 1] -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] - -(21) HashAggregate [codegen id : 1] -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +Input [2]: [cc_call_center_sk#11, cc_name#12] +Condition : isnotnull(cc_call_center_sk#11) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cc_call_center_sk#11, cc_name#12] + +(17) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cs_call_center_sk#2], [cc_call_center_sk#11], Inner, BuildRight + +(18) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_call_center_sk#11, cc_name#12] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] + +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) + +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(23) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight + +(24) CometProject +Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12, d_date_sk#13] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] + +(25) CometHashAggregate +Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(22) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 2] -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(24) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,cc_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,cc_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/extended.txt new file mode 100644 index 0000000000..c075163849 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.ship_mode + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/simplified.txt index 626d31bc0b..98a215dbc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/simplified.txt @@ -1,28 +1,31 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (2) - HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,cc_name] #1 - WholeStageCodegen (1) - HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] - CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] - CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [cc_call_center_sk,cc_name] #2 - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,cc_name] #1 + CometHashAggregate [cs_ship_date_sk,cs_sold_date_sk] [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] + CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] + CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [cc_call_center_sk,cc_name] #4 + CometFilter [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/explain.txt index 2836e4c129..44a03d1b96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/explain.txt @@ -1,262 +1,234 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Filter (14) - : : : +- * HashAggregate (13) - : : : +- Exchange (12) - : : : +- * HashAggregate (11) - : : : +- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_returns (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * HashAggregate (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store_returns (15) - : : +- ReusedExchange (18) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.store (31) - +- BroadcastExchange (41) - +- * Filter (40) - +- * ColumnarToRow (39) - +- Scan parquet spark_catalog.default.customer (38) - - -(1) Scan parquet spark_catalog.default.store_returns +* CometColumnarToRow (41) ++- CometTakeOrderedAndProject (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometFilter (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (25) + : : +- CometFilter (24) + : : +- CometHashAggregate (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometHashAggregate (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (13) + : : +- ReusedExchange (15) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.customer (34) + + +(1) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_year#6] Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [sr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 2] -Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] +(8) CometProject Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] +Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -(11) HashAggregate [codegen id : 2] +(9) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -(12) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) HashAggregate [codegen id : 9] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +(11) CometHashAggregate +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#10, sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] -(14) Filter [codegen id : 9] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) +(12) CometFilter +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Condition : isnotnull(ctr_total_return#10) -(15) Scan parquet spark_catalog.default.store_returns +(13) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(17) Filter [codegen id : 4] +(14) CometFilter Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Condition : isnotnull(sr_store_sk#2) -(18) ReusedExchange [Reuses operator id: 8] +(15) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#5] -(19) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [sr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(20) Project [codegen id : 4] -Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] +(17) CometProject Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] +Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -(21) HashAggregate [codegen id : 4] +(18) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#13] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] -(22) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(19) CometExchange +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#11] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(23) HashAggregate [codegen id : 5] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] +(20) CometHashAggregate +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#11] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [2]: [sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] - -(24) HashAggregate [codegen id : 5] -Input [2]: [ctr_store_sk#11, ctr_total_return#12] -Keys [1]: [ctr_store_sk#11] -Functions [1]: [partial_avg(ctr_total_return#12)] -Aggregate Attributes [2]: [sum#15, count#16] -Results [3]: [ctr_store_sk#11, sum#17, count#18] - -(25) Exchange -Input [3]: [ctr_store_sk#11, sum#17, count#18] -Arguments: hashpartitioning(ctr_store_sk#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 6] -Input [3]: [ctr_store_sk#11, sum#17, count#18] -Keys [1]: [ctr_store_sk#11] -Functions [1]: [avg(ctr_total_return#12)] -Aggregate Attributes [1]: [avg(ctr_total_return#12)#19] -Results [2]: [(avg(ctr_total_return#12)#19 * 1.2) AS (avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11 AS ctr_store_sk#11#21] - -(27) Filter [codegen id : 6] -Input [2]: [(avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#20) - -(28) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=5] - -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [ctr_store_sk#11#21] -Join type: Inner -Join condition: (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#20) - -(30) Project [codegen id : 9] -Output [2]: [ctr_customer_sk#10, ctr_store_sk#11] -Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] - -(31) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#22, s_state#23] + +(21) CometHashAggregate +Input [2]: [ctr_store_sk#9, ctr_total_return#10] +Keys [1]: [ctr_store_sk#9] +Functions [1]: [partial_avg(ctr_total_return#10)] + +(22) CometExchange +Input [3]: [ctr_store_sk#9, sum#12, count#13] +Arguments: hashpartitioning(ctr_store_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(23) CometHashAggregate +Input [3]: [ctr_store_sk#9, sum#12, count#13] +Keys [1]: [ctr_store_sk#9] +Functions [1]: [avg(ctr_total_return#10)] + +(24) CometFilter +Input [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#14) + +(25) CometBroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] + +(26) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Right output [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [ctr_store_sk#9], [ctr_store_sk#9#15], Inner, (cast(ctr_total_return#10 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#14), BuildRight + +(27) CometProject +Input [5]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10, (avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [ctr_customer_sk#8, ctr_store_sk#9], [ctr_customer_sk#8, ctr_store_sk#9] + +(28) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#16, s_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#22, s_state#23] +(29) CometFilter +Input [2]: [s_store_sk#16, s_state#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) = TN) AND isnotnull(s_store_sk#16)) -(33) Filter [codegen id : 7] -Input [2]: [s_store_sk#22, s_state#23] -Condition : ((isnotnull(s_state#23) AND (s_state#23 = TN)) AND isnotnull(s_store_sk#22)) +(30) CometProject +Input [2]: [s_store_sk#16, s_state#17] +Arguments: [s_store_sk#16], [s_store_sk#16] -(34) Project [codegen id : 7] -Output [1]: [s_store_sk#22] -Input [2]: [s_store_sk#22, s_state#23] +(31) CometBroadcastExchange +Input [1]: [s_store_sk#16] +Arguments: [s_store_sk#16] -(35) BroadcastExchange -Input [1]: [s_store_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(32) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#8, ctr_store_sk#9] +Right output [1]: [s_store_sk#16] +Arguments: [ctr_store_sk#9], [s_store_sk#16], Inner, BuildRight -(36) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [s_store_sk#22] -Join type: Inner -Join condition: None +(33) CometProject +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, s_store_sk#16] +Arguments: [ctr_customer_sk#8], [ctr_customer_sk#8] -(37) Project [codegen id : 9] -Output [1]: [ctr_customer_sk#10] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, s_store_sk#22] - -(38) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#24, c_customer_id#25] +(34) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#18, c_customer_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 8] -Input [2]: [c_customer_sk#24, c_customer_id#25] +(35) CometFilter +Input [2]: [c_customer_sk#18, c_customer_id#19] +Condition : isnotnull(c_customer_sk#18) + +(36) CometProject +Input [2]: [c_customer_sk#18, c_customer_id#19] +Arguments: [c_customer_sk#18, c_customer_id#20], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#19, 16, true, false, true) AS c_customer_id#20] -(40) Filter [codegen id : 8] -Input [2]: [c_customer_sk#24, c_customer_id#25] -Condition : isnotnull(c_customer_sk#24) +(37) CometBroadcastExchange +Input [2]: [c_customer_sk#18, c_customer_id#20] +Arguments: [c_customer_sk#18, c_customer_id#20] -(41) BroadcastExchange -Input [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(38) CometBroadcastHashJoin +Left output [1]: [ctr_customer_sk#8] +Right output [2]: [c_customer_sk#18, c_customer_id#20] +Arguments: [ctr_customer_sk#8], [c_customer_sk#18], Inner, BuildRight -(42) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_customer_sk#10] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None +(39) CometProject +Input [3]: [ctr_customer_sk#8, c_customer_sk#18, c_customer_id#20] +Arguments: [c_customer_id#20], [c_customer_id#20] -(43) Project [codegen id : 9] -Output [1]: [c_customer_id#25] -Input [3]: [ctr_customer_sk#10, c_customer_sk#24, c_customer_id#25] +(40) CometTakeOrderedAndProject +Input [1]: [c_customer_id#20] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#20 ASC NULLS FIRST], output=[c_customer_id#20]), [c_customer_id#20], 100, 0, [c_customer_id#20 ASC NULLS FIRST], [c_customer_id#20] -(44) TakeOrderedAndProject -Input [1]: [c_customer_id#25] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST], [c_customer_id#25] +(41) CometColumnarToRow [codegen id : 1] +Input [1]: [c_customer_id#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/extended.txt new file mode 100644 index 0000000000..9470bbb18b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/extended.txt @@ -0,0 +1,46 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 43 out of 43 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/simplified.txt index 0e1c105b70..b37ce05698 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/simplified.txt @@ -1,65 +1,43 @@ -TakeOrderedAndProject [c_customer_id] - WholeStageCodegen (9) - Project [c_customer_id] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk] - BroadcastHashJoin [ctr_store_sk,s_store_sk] - Project [ctr_customer_sk,ctr_store_sk] - BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #1 - WholeStageCodegen (2) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - Project [sr_customer_sk,sr_store_sk,sr_return_amt] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk,sr_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (6) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count] - InputAdapter - Exchange [ctr_store_sk] #4 - WholeStageCodegen (5) - HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #5 - WholeStageCodegen (4) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - Project [sr_customer_sk,sr_store_sk,sr_return_amt] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_customer_id] + CometProject [c_customer_id] + CometBroadcastHashJoin [ctr_customer_sk,c_customer_sk,c_customer_id] + CometProject [ctr_customer_sk] + CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,s_store_sk] + CometProject [ctr_customer_sk,ctr_store_sk] + CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_store_sk] + CometFilter [ctr_customer_sk,ctr_store_sk,ctr_total_return] + CometHashAggregate [sum] [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum(UnscaledValue(sr_return_amt))] + CometExchange [sr_customer_sk,sr_store_sk] #1 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #3 + CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_store_sk,avg(ctr_total_return),ctr_store_sk] + CometExchange [ctr_store_sk] #4 + CometHashAggregate [ctr_total_return] [ctr_store_sk,sum,count] + CometHashAggregate [sr_customer_sk,sum] [ctr_store_sk,ctr_total_return,sr_store_sk,sum(UnscaledValue(sr_return_amt))] + CometExchange [sr_customer_sk,sr_store_sk] #5 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + CometBroadcastExchange [s_store_sk] #6 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [c_customer_sk,c_customer_id] #7 + CometProject [c_customer_id] [c_customer_sk,c_customer_id] + CometFilter [c_customer_sk,c_customer_id] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/explain.txt index 10320c7c3c..a9f1f31337 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/explain.txt @@ -1,272 +1,269 @@ == Physical Plan == TakeOrderedAndProject (47) +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (41) - +- * Filter (40) - +- * ColumnarToRow (39) - +- Scan parquet spark_catalog.default.customer_demographics (38) - - -(1) Scan parquet spark_catalog.default.customer + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] +(6) CometProject Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] +(9) CometProject Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#6] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#13] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None -(29) Filter [codegen id : 9] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(30) Project [codegen id : 9] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_county#18] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_county#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] +(30) CometFilter +Input [2]: [ca_address_sk#15, ca_county#16] +Condition : (ca_county#16 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#15)) -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] -Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) +(31) CometProject +Input [2]: [ca_address_sk#15, ca_county#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] -(34) Project [codegen id : 7] -Output [1]: [ca_address_sk#17] -Input [2]: [ca_address_sk#17, ca_county#18] +(32) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#15] -(35) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(33) BroadcastExchange +Input [1]: [ca_address_sk#15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(36) BroadcastHashJoin [codegen id : 9] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(37) Project [codegen id : 9] +(35) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15] -(38) Scan parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(37) CometFilter +Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#17) -(40) Filter [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#19) +(38) CometProject +Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#18, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#28, cd_purchase_estimate#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#22, 10, true, false, true) AS cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(41) BroadcastExchange -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(39) CometColumnarToRow [codegen id : 4] +Input [9]: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(42) BroadcastHashJoin [codegen id : 9] +(40) BroadcastExchange +Input [9]: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: None -(43) Project [codegen id : 9] -Output [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(42) Project [codegen id : 5] +Output [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(44) HashAggregate [codegen id : 9] -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(43) HashAggregate [codegen id : 5] +Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] +Aggregate Attributes [1]: [count#30] +Results [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] + +(44) CometColumnarExchange +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] +Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(45) Exchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(45) CometColumnarToRow [codegen id : 6] +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] -(46) HashAggregate [codegen id : 10] -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(46) HashAggregate [codegen id : 6] +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] +Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, count(1)#30 AS cnt1#31, cd_purchase_estimate#23, count(1)#30 AS cnt2#32, cd_credit_rating#24, count(1)#30 AS cnt3#33, cd_dep_count#25, count(1)#30 AS cnt4#34, cd_dep_employed_count#26, count(1)#30 AS cnt5#35, cd_dep_college_count#27, count(1)#30 AS cnt6#36] +Aggregate Attributes [1]: [count(1)#32] +Results [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#32 AS cnt1#33, cd_purchase_estimate#21, count(1)#32 AS cnt2#34, cd_credit_rating#29, count(1)#32 AS cnt3#35, cd_dep_count#23, count(1)#32 AS cnt4#36, cd_dep_employed_count#24, count(1)#32 AS cnt5#37, cd_dep_college_count#25, count(1)#32 AS cnt6#38] (47) TakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] -Arguments: 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] +Input [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#21, cnt2#34, cd_credit_rating#29, cnt3#35, cd_dep_count#23, cnt4#36, cd_dep_employed_count#24, cnt5#37, cd_dep_college_count#25, cnt6#38] +Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#29 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#21, cnt2#34, cd_credit_rating#29, cnt3#35, cd_dep_count#23, cnt4#36, cd_dep_employed_count#24, cnt5#37, cd_dep_college_count#25, cnt6#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/simplified.txt index bd3d7f63ba..e5aeaa20c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/simplified.txt @@ -1,71 +1,63 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/explain.txt index 994ad18545..d9afb2f640 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/explain.txt @@ -1,423 +1,388 @@ == Physical Plan == -TakeOrderedAndProject (72) -+- * Project (71) - +- * BroadcastHashJoin Inner BuildRight (70) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * HashAggregate (52) - : +- Exchange (51) - : +- * HashAggregate (50) - : +- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (46) - : : +- * BroadcastHashJoin Inner BuildRight (45) - : : :- * Filter (40) - : : : +- * ColumnarToRow (39) - : : : +- Scan parquet spark_catalog.default.customer (38) - : : +- BroadcastExchange (44) - : : +- * Filter (43) - : : +- * ColumnarToRow (42) - : : +- Scan parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (47) - +- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Filter (59) - : : +- * ColumnarToRow (58) - : : +- Scan parquet spark_catalog.default.customer (57) - : +- ReusedExchange (60) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (69) ++- CometTakeOrderedAndProject (68) + +- CometProject (67) + +- CometBroadcastHashJoin (66) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (34) + : : +- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (50) + : +- CometFilter (49) + : +- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometProject (37) + : : : +- CometFilter (36) + : : : +- CometNativeScan parquet spark_catalog.default.customer (35) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- CometBroadcastExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometProject (55) + : : +- CometFilter (54) + : : +- CometNativeScan parquet spark_catalog.default.customer (53) + : +- ReusedExchange (56) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) -(6) Filter [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(7) BroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] - -(16) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(17) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(23) ReusedExchange [Reuses operator id: 7] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#20] -Right keys [1]: [ss_customer_sk#28] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(25) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#31] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] - -(32) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(33) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [3]: [c_customer_id#21 AS customer_id#36, c_preferred_cust_flag#24 AS customer_preferred_cust_flag#37, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#38] - -(35) BroadcastExchange -Input [3]: [customer_id#36, customer_preferred_cust_flag#37, year_total#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 16] -Output [4]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38] -Input [5]: [customer_id#18, year_total#19, customer_id#36, customer_preferred_cust_flag#37, year_total#38] - -(38) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] + +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight + +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] + +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometProject +Input [5]: [customer_id#22, year_total#23, customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] + +(35) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] +(36) CometFilter +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Condition : (isnotnull(c_customer_sk#42) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true))) -(40) Filter [codegen id : 10] -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_customer_id#40)) +(37) CometProject +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Arguments: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55], [c_customer_sk#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#44, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#45, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#46, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#48, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#49, 50, true, false, true) AS c_email_address#55] -(41) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] +(38) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#50)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#59)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 8] -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(43) Filter [codegen id : 8] -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Condition : isnotnull(ws_bill_customer_sk#47) - -(44) BroadcastExchange -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(45) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#39] -Right keys [1]: [ws_bill_customer_sk#47] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 10] -Output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Input [12]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(47) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#51, d_year#52] - -(48) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(49) Project [codegen id : 10] -Output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Input [12]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(50) HashAggregate [codegen id : 10] -Input [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum#53] -Results [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] - -(51) Exchange -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) HashAggregate [codegen id : 11] -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55] -Results [2]: [c_customer_id#40 AS customer_id#56, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55,18,2) AS year_total#57] - -(53) Filter [codegen id : 11] -Input [2]: [customer_id#56, year_total#57] -Condition : (isnotnull(year_total#57) AND (year_total#57 > 0.00)) - -(54) BroadcastExchange -Input [2]: [customer_id#56, year_total#57] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(55) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#56] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 16] -Output [5]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57] -Input [6]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, customer_id#56, year_total#57] - -(57) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] +(39) CometFilter +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Condition : isnotnull(ws_bill_customer_sk#56) + +(40) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] + +(41) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#42], [ws_bill_customer_sk#56], Inner, BuildRight + +(42) CometProject +Input [12]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] + +(43) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#60, d_year#61] + +(44) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(45) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] + +(46) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] + +(47) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(48) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] + +(49) CometFilter +Input [2]: [customer_id#63, year_total#64] +Condition : (isnotnull(year_total#64) AND (year_total#64 > 0.00)) + +(50) CometBroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#63, year_total#64] + +(51) CometBroadcastHashJoin +Left output [4]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] +Right output [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#22], [customer_id#63], Inner, BuildRight + +(52) CometProject +Input [6]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, customer_id#63, year_total#64] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] + +(53) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] - -(59) Filter [codegen id : 14] -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Condition : (isnotnull(c_customer_sk#58) AND isnotnull(c_customer_id#59)) - -(60) ReusedExchange [Reuses operator id: 44] -Output [4]: [ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(61) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#58] -Right keys [1]: [ws_bill_customer_sk#66] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 14] -Output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Input [12]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(63) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#70, d_year#71] - -(64) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#69] -Right keys [1]: [d_date_sk#70] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 14] -Output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69, d_date_sk#70, d_year#71] - -(66) HashAggregate [codegen id : 14] -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum#72] -Results [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] - -(67) Exchange -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 15] -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55] -Results [2]: [c_customer_id#59 AS customer_id#74, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55,18,2) AS year_total#75] - -(69) BroadcastExchange -Input [2]: [customer_id#74, year_total#75] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(70) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#74] -Join type: Inner -Join condition: (CASE WHEN (year_total#57 > 0.00) THEN (year_total#75 / year_total#57) END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#38 / year_total#19) END) - -(71) Project [codegen id : 16] -Output [1]: [customer_preferred_cust_flag#37] -Input [7]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57, customer_id#74, year_total#75] - -(72) TakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#37] -Arguments: 100, [customer_preferred_cust_flag#37 ASC NULLS FIRST], [customer_preferred_cust_flag#37] +(54) CometFilter +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Condition : (isnotnull(c_customer_sk#65) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true))) + +(55) CometProject +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Arguments: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55], [c_customer_sk#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#67, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#68, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#69, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#71, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#72, 50, true, false, true) AS c_email_address#55] + +(56) ReusedExchange [Reuses operator id: 40] +Output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] + +(57) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_sk#65], [ws_bill_customer_sk#73], Inner, BuildRight + +(58) CometProject +Input [12]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] + +(59) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#77, d_year#78] + +(60) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Right output [2]: [d_date_sk#77, d_year#78] +Arguments: [ws_sold_date_sk#76], [d_date_sk#77], Inner, BuildRight + +(61) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76, d_date_sk#77, d_year#78] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] + +(62) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] + +(63) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(64) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] + +(65) CometBroadcastExchange +Input [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#80, year_total#81] + +(66) CometBroadcastHashJoin +Left output [5]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] +Right output [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#22], [customer_id#80], Inner, (CASE WHEN (year_total#64 > 0.00) THEN (year_total#81 / year_total#64) END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#41 / year_total#23) END), BuildRight + +(67) CometProject +Input [7]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64, customer_id#80, year_total#81] +Arguments: [customer_preferred_cust_flag#40], [customer_preferred_cust_flag#40] + +(68) CometTakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#40] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#40 ASC NULLS FIRST], output=[customer_preferred_cust_flag#40]), [customer_preferred_cust_flag#40], 100, 0, [customer_preferred_cust_flag#40 ASC NULLS FIRST], [customer_preferred_cust_flag#40] + +(69) CometColumnarToRow [codegen id : 1] +Input [1]: [customer_preferred_cust_flag#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/extended.txt new file mode 100644 index 0000000000..1c5f33227c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/extended.txt @@ -0,0 +1,79 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 76 out of 76 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/simplified.txt index 944e89eee5..b69b7cc000 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/simplified.txt @@ -1,107 +1,71 @@ -TakeOrderedAndProject [customer_preferred_cust_flag] - WholeStageCodegen (16) - Project [customer_preferred_cust_flag] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_preferred_cust_flag] + CometProject [customer_preferred_cust_flag] + CometBroadcastHashJoin [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_preferred_cust_flag,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_preferred_cust_flag,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #4 + CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/explain.txt index d70125ed20..7fff9bfcea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/explain.txt @@ -1,140 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/extended.txt new file mode 100644 index 0000000000..9b405bc9ee --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/simplified.txt index 3bbeaadc8f..3f383800c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/explain.txt index fcf177544f..260585cee3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/explain.txt @@ -1,222 +1,198 @@ == Physical Plan == -* HashAggregate (38) -+- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store (4) - : : : +- BroadcastExchange (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : +- BroadcastExchange (21) - : : +- * Project (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.date_dim (17) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.customer_demographics (24) - +- BroadcastExchange (33) - +- * Filter (32) - +- * ColumnarToRow (31) - +- Scan parquet spark_catalog.default.household_demographics (30) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (34) ++- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.store (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) + +- CometBroadcastExchange (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.household_demographics (26) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(4) Scan parquet spark_catalog.default.store +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [s_store_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [s_store_sk#11] Condition : isnotnull(s_store_sk#11) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [s_store_sk#11] -(8) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight -(9) Project [codegen id : 6] -Output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +(7) CometProject Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [OH,TX]),In(ca_state, [KY,NM,OR])),In(ca_state, [MS,TX,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(9) CometFilter Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (VA,TX,MS))) -(12) Filter [codegen id : 2] +(10) CometProject Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) AS ca_state#15] -(13) Project [codegen id : 2] -Output [2]: [ca_address_sk#12, ca_state#13] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +(11) CometBroadcastExchange +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] -(14) BroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight -(15) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#12] -Join type: Inner -Join condition: ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))) +(13) CometProject +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(16) Project [codegen id : 6] -Output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] - -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#15, d_year#16] - -(19) Filter [codegen id : 3] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(15) CometFilter +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(20) Project [codegen id : 3] -Output [1]: [d_date_sk#15] -Input [2]: [d_date_sk#15, d_year#16] +(16) CometProject +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] -(21) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight -(23) Project [codegen id : 6] -Output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] +(19) CometProject +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(24) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(20) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(21) CometFilter +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(26) Filter [codegen id : 4] -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) +(22) CometProject +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(27) BroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))) +(24) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight -(29) Project [codegen id : 6] -Output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(25) CometProject +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] -(30) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_dep_count#21] +(26) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#23, hd_dep_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_dep_count#21] +(27) CometFilter +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) -(32) Filter [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) +(28) CometBroadcastExchange +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] -(33) BroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#20] -Join type: Inner -Join condition: (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))) +(30) CometProject +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(35) Project [codegen id : 6] -Output [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] - -(36) HashAggregate [codegen id : 6] +(31) CometHashAggregate Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Results [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -(37) Exchange -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometExchange +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(38) HashAggregate [codegen id : 7] -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(33) CometHashAggregate +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [4]: [avg(ss_quantity#5)#36, avg(UnscaledValue(ss_ext_sales_price#7))#37, avg(UnscaledValue(ss_ext_wholesale_cost#8))#38, sum(UnscaledValue(ss_ext_wholesale_cost#8))#39] -Results [4]: [avg(ss_quantity#5)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledValue(ss_ext_sales_price#7))#37 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#41, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#38 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#42, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#39,17,2) AS sum(ss_ext_wholesale_cost)#43] + +(34) CometColumnarToRow [codegen id : 1] +Input [4]: [avg(ss_quantity)#32, avg(ss_ext_sales_price)#33, avg(ss_ext_wholesale_cost)#34, sum(ss_ext_wholesale_cost)#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/extended.txt new file mode 100644 index 0000000000..b69894e484 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/extended.txt @@ -0,0 +1,36 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_demographics + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.household_demographics + +Comet accelerated 33 out of 33 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/simplified.txt index 7c63e82534..bd03c2152d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/simplified.txt @@ -1,57 +1,36 @@ -WholeStageCodegen (7) - HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum] - Project [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price,hd_dep_count] - Project [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] - BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_addr_sk,ss_cdemo_sk,ss_hdemo_sk,ss_net_profit,ss_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [hd_demo_sk,hd_dep_count] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometHashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] + CometExchange #1 + CometHashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum] + CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] + CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 + CometFilter [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/explain.txt index aee3f209cf..f1b4ed4fe3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/explain.txt @@ -1,772 +1,680 @@ == Physical Plan == -TakeOrderedAndProject (113) -+- * HashAggregate (112) - +- Exchange (111) - +- * HashAggregate (110) - +- * Expand (109) - +- Union (108) - :- * Project (75) - : +- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - :- * Project (91) - : +- * Filter (90) - : +- * HashAggregate (89) - : +- Exchange (88) - : +- * HashAggregate (87) - : +- * Project (86) - : +- * BroadcastHashJoin Inner BuildRight (85) - : :- * Project (83) - : : +- * BroadcastHashJoin Inner BuildRight (82) - : : :- * BroadcastHashJoin LeftSemi BuildRight (80) - : : : :- * Filter (78) - : : : : +- * ColumnarToRow (77) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (76) - : : : +- ReusedExchange (79) - : : +- ReusedExchange (81) - : +- ReusedExchange (84) - +- * Project (107) - +- * Filter (106) - +- * HashAggregate (105) - +- Exchange (104) - +- * HashAggregate (103) - +- * Project (102) - +- * BroadcastHashJoin Inner BuildRight (101) - :- * Project (99) - : +- * BroadcastHashJoin Inner BuildRight (98) - : :- * BroadcastHashJoin LeftSemi BuildRight (96) - : : :- * Filter (94) - : : : +- * ColumnarToRow (93) - : : : +- Scan parquet spark_catalog.default.web_sales (92) - : : +- ReusedExchange (95) - : +- ReusedExchange (97) - +- ReusedExchange (100) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (102) ++- CometTakeOrderedAndProject (101) + +- CometHashAggregate (100) + +- CometExchange (99) + +- CometHashAggregate (98) + +- CometExpand (97) + +- CometUnion (96) + :- CometProject (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + :- CometProject (80) + : +- CometFilter (79) + : +- CometHashAggregate (78) + : +- CometExchange (77) + : +- CometHashAggregate (76) + : +- CometProject (75) + : +- CometBroadcastHashJoin (74) + : :- CometProject (72) + : : +- CometBroadcastHashJoin (71) + : : :- CometBroadcastHashJoin (69) + : : : :- CometFilter (67) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (66) + : : : +- ReusedExchange (68) + : : +- ReusedExchange (70) + : +- ReusedExchange (73) + +- CometProject (95) + +- CometFilter (94) + +- CometHashAggregate (93) + +- CometExchange (92) + +- CometHashAggregate (91) + +- CometProject (90) + +- CometBroadcastHashJoin (89) + :- CometProject (87) + : +- CometBroadcastHashJoin (86) + : :- CometBroadcastHashJoin (84) + : : :- CometFilter (82) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (85) + +- ReusedExchange (88) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : isnotnull(i_item_sk#35) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [3]: [d_date_sk#39, d_year#40, d_moy#41] Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#42, isEmpty#43, count#44] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 26] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48, count(1)#49] -Results [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48 AS sales#50, count(1)#49 AS number_sales#51] -(74) Filter [codegen id : 26] -Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#50, number_sales#51] -Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +(64) CometFilter +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) -(75) Project [codegen id : 26] -Output [6]: [sales#50, number_sales#51, store AS channel#54, i_brand_id#36 AS i_brand_id#55, i_class_id#37 AS i_class_id#56, i_category_id#38 AS i_category_id#57] -Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#50, number_sales#51] +(65) CometProject +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] -(76) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] +(66) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#61)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(77) ColumnarToRow [codegen id : 51] -Input [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] - -(78) Filter [codegen id : 51] -Input [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] -Condition : isnotnull(cs_item_sk#58) +(67) CometFilter +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) -(79) ReusedExchange [Reuses operator id: 54] +(68) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(80) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#58] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(81) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] - -(82) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#58] -Right keys [1]: [i_item_sk#62] -Join type: Inner -Join condition: None - -(83) Project [codegen id : 51] -Output [6]: [cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [8]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] - -(84) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#66] - -(85) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_sold_date_sk#61] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 51] -Output [5]: [cs_quantity#59, cs_list_price#60, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [7]: [cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_brand_id#63, i_class_id#64, i_category_id#65, d_date_sk#66] - -(87) HashAggregate [codegen id : 51] -Input [5]: [cs_quantity#59, cs_list_price#60, i_brand_id#63, i_class_id#64, i_category_id#65] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [partial_sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60)), partial_count(1)] -Aggregate Attributes [3]: [sum#67, isEmpty#68, count#69] -Results [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] - -(88) Exchange -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] -Arguments: hashpartitioning(i_brand_id#63, i_class_id#64, i_category_id#65, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(89) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60))#73, count(1)#74] -Results [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60))#73 AS sales#75, count(1)#74 AS number_sales#76] - -(90) Filter [codegen id : 52] -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#75, number_sales#76] -Condition : (isnotnull(sales#75) AND (cast(sales#75 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) - -(91) Project [codegen id : 52] -Output [6]: [sales#75, number_sales#76, catalog AS channel#77, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#75, number_sales#76] - -(92) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] +(69) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#34] +Arguments: [cs_item_sk#53], [ss_item_sk#34], LeftSemi, BuildRight + +(70) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] + +(71) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] +Arguments: [cs_item_sk#53], [i_item_sk#57], Inner, BuildRight + +(72) CometProject +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60] + +(73) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#61] + +(74) CometBroadcastHashJoin +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60] +Right output [1]: [d_date_sk#61] +Arguments: [cs_sold_date_sk#56], [d_date_sk#61], Inner, BuildRight + +(75) CometProject +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60, d_date_sk#61] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60], [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60] + +(76) CometHashAggregate +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60] +Keys [3]: [i_brand_id#58, i_class_id#59, i_category_id#60] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] + +(77) CometExchange +Input [6]: [i_brand_id#58, i_class_id#59, i_category_id#60, sum#62, isEmpty#63, count#64] +Arguments: hashpartitioning(i_brand_id#58, i_class_id#59, i_category_id#60, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(78) CometHashAggregate +Input [6]: [i_brand_id#58, i_class_id#59, i_category_id#60, sum#62, isEmpty#63, count#64] +Keys [3]: [i_brand_id#58, i_class_id#59, i_category_id#60] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] + +(79) CometFilter +Input [5]: [i_brand_id#58, i_class_id#59, i_category_id#60, sales#65, number_sales#66] +Condition : (isnotnull(sales#65) AND (cast(sales#65 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(80) CometProject +Input [5]: [i_brand_id#58, i_class_id#59, i_category_id#60, sales#65, number_sales#66] +Arguments: [sales#65, number_sales#66, channel#67, i_brand_id#58, i_class_id#59, i_category_id#60], [sales#65, number_sales#66, catalog AS channel#67, i_brand_id#58, i_class_id#59, i_category_id#60] + +(81) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#71)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(93) ColumnarToRow [codegen id : 77] -Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] - -(94) Filter [codegen id : 77] -Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_item_sk#78) +(82) CometFilter +Input [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Condition : isnotnull(ws_item_sk#68) -(95) ReusedExchange [Reuses operator id: 54] +(83) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(96) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#78] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(97) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#82, i_brand_id#83, i_class_id#84, i_category_id#85] - -(98) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#78] -Right keys [1]: [i_item_sk#82] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 77] -Output [6]: [ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [8]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_item_sk#82, i_brand_id#83, i_class_id#84, i_category_id#85] - -(100) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#86] - -(101) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_sold_date_sk#81] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 77] -Output [5]: [ws_quantity#79, ws_list_price#80, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [7]: [ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_brand_id#83, i_class_id#84, i_category_id#85, d_date_sk#86] - -(103) HashAggregate [codegen id : 77] -Input [5]: [ws_quantity#79, ws_list_price#80, i_brand_id#83, i_class_id#84, i_category_id#85] -Keys [3]: [i_brand_id#83, i_class_id#84, i_category_id#85] -Functions [2]: [partial_sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80)), partial_count(1)] -Aggregate Attributes [3]: [sum#87, isEmpty#88, count#89] -Results [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] - -(104) Exchange -Input [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] -Arguments: hashpartitioning(i_brand_id#83, i_class_id#84, i_category_id#85, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(105) HashAggregate [codegen id : 78] -Input [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] -Keys [3]: [i_brand_id#83, i_class_id#84, i_category_id#85] -Functions [2]: [sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80))#93, count(1)#94] -Results [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80))#93 AS sales#95, count(1)#94 AS number_sales#96] - -(106) Filter [codegen id : 78] -Input [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sales#95, number_sales#96] -Condition : (isnotnull(sales#95) AND (cast(sales#95 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) - -(107) Project [codegen id : 78] -Output [6]: [sales#95, number_sales#96, web AS channel#97, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sales#95, number_sales#96] - -(108) Union - -(109) Expand [codegen id : 79] -Input [6]: [sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, i_category_id#57] -Arguments: [[sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, i_category_id#57, 0], [sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, null, 1], [sales#50, number_sales#51, channel#54, i_brand_id#55, null, null, 3], [sales#50, number_sales#51, channel#54, null, null, null, 7], [sales#50, number_sales#51, null, null, null, null, 15]], [sales#50, number_sales#51, channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] - -(110) HashAggregate [codegen id : 79] -Input [7]: [sales#50, number_sales#51, channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Keys [5]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] -Aggregate Attributes [3]: [sum#103, isEmpty#104, sum#105] -Results [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] - -(111) Exchange -Input [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] -Arguments: hashpartitioning(channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(112) HashAggregate [codegen id : 80] -Input [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] -Keys [5]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Functions [2]: [sum(sales#50), sum(number_sales#51)] -Aggregate Attributes [2]: [sum(sales#50)#109, sum(number_sales#51)#110] -Results [6]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales#50)#109 AS sum(sales)#111, sum(number_sales#51)#110 AS sum(number_sales)#112] - -(113) TakeOrderedAndProject -Input [6]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales)#111, sum(number_sales)#112] -Arguments: 100, [channel#98 ASC NULLS FIRST, i_brand_id#99 ASC NULLS FIRST, i_class_id#100 ASC NULLS FIRST, i_category_id#101 ASC NULLS FIRST], [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales)#111, sum(number_sales)#112] +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Right output [1]: [ss_item_sk#34] +Arguments: [ws_item_sk#68], [ss_item_sk#34], LeftSemi, BuildRight + +(85) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] + +(86) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Right output [4]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] +Arguments: [ws_item_sk#68], [i_item_sk#72], Inner, BuildRight + +(87) CometProject +Input [8]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] +Arguments: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75], [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75] + +(88) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#76] + +(89) CometBroadcastHashJoin +Left output [6]: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75] +Right output [1]: [d_date_sk#76] +Arguments: [ws_sold_date_sk#71], [d_date_sk#76], Inner, BuildRight + +(90) CometProject +Input [7]: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, d_date_sk#76] +Arguments: [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75], [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75] + +(91) CometHashAggregate +Input [5]: [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75] +Keys [3]: [i_brand_id#73, i_class_id#74, i_category_id#75] +Functions [2]: [partial_sum((cast(ws_quantity#69 as decimal(10,0)) * ws_list_price#70)), partial_count(1)] + +(92) CometExchange +Input [6]: [i_brand_id#73, i_class_id#74, i_category_id#75, sum#77, isEmpty#78, count#79] +Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(93) CometHashAggregate +Input [6]: [i_brand_id#73, i_class_id#74, i_category_id#75, sum#77, isEmpty#78, count#79] +Keys [3]: [i_brand_id#73, i_class_id#74, i_category_id#75] +Functions [2]: [sum((cast(ws_quantity#69 as decimal(10,0)) * ws_list_price#70)), count(1)] + +(94) CometFilter +Input [5]: [i_brand_id#73, i_class_id#74, i_category_id#75, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(95) CometProject +Input [5]: [i_brand_id#73, i_class_id#74, i_category_id#75, sales#80, number_sales#81] +Arguments: [sales#80, number_sales#81, channel#82, i_brand_id#73, i_class_id#74, i_category_id#75], [sales#80, number_sales#81, web AS channel#82, i_brand_id#73, i_class_id#74, i_category_id#75] + +(96) CometUnion +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#65, number_sales#66, channel#67, i_brand_id#58, i_class_id#59, i_category_id#60] +Child 2 Input [6]: [sales#80, number_sales#81, channel#82, i_brand_id#73, i_class_id#74, i_category_id#75] + +(97) CometExpand +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] + +(98) CometHashAggregate +Input [7]: [sales#45, number_sales#46, channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Keys [5]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] + +(99) CometExchange +Input [8]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, sum#88, isEmpty#89, sum#90] +Arguments: hashpartitioning(channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(100) CometHashAggregate +Input [8]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, sum#88, isEmpty#89, sum#90] +Keys [5]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Functions [2]: [sum(sales#45), sum(number_sales#46)] + +(101) CometTakeOrderedAndProject +Input [6]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#83 ASC NULLS FIRST,i_brand_id#84 ASC NULLS FIRST,i_class_id#85 ASC NULLS FIRST,i_category_id#86 ASC NULLS FIRST], output=[channel#83,i_brand_id#84,i_class_id#85,i_category_id#86,sum(sales)#91,sum(number_sales)#92]), [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92], 100, 0, [channel#83 ASC NULLS FIRST, i_brand_id#84 ASC NULLS FIRST, i_class_id#85 ASC NULLS FIRST, i_category_id#86 ASC NULLS FIRST], [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] + +(102) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#52, [id=#53] -* HashAggregate (132) -+- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * Project (118) - : +- * BroadcastHashJoin Inner BuildRight (117) - : :- * ColumnarToRow (115) - : : +- Scan parquet spark_catalog.default.store_sales (114) - : +- ReusedExchange (116) - :- * Project (123) - : +- * BroadcastHashJoin Inner BuildRight (122) - : :- * ColumnarToRow (120) - : : +- Scan parquet spark_catalog.default.catalog_sales (119) - : +- ReusedExchange (121) - +- * Project (128) - +- * BroadcastHashJoin Inner BuildRight (127) - :- * ColumnarToRow (125) - : +- Scan parquet spark_catalog.default.web_sales (124) - +- ReusedExchange (126) - - -(114) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* CometColumnarToRow (119) ++- CometHashAggregate (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometUnion (115) + :- CometProject (106) + : +- CometBroadcastHashJoin (105) + : :- CometNativeScan parquet spark_catalog.default.store_sales (103) + : +- ReusedExchange (104) + :- CometProject (110) + : +- CometBroadcastHashJoin (109) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (107) + : +- ReusedExchange (108) + +- CometProject (114) + +- CometBroadcastHashJoin (113) + :- CometNativeScan parquet spark_catalog.default.web_sales (111) + +- ReusedExchange (112) + + +(103) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#115)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#95)] ReadSchema: struct -(115) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115] - -(116) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#116] +(104) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#96] -(117) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#116] -Join type: Inner -Join condition: None +(105) CometBroadcastHashJoin +Left output [3]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95] +Right output [1]: [d_date_sk#96] +Arguments: [ss_sold_date_sk#95], [d_date_sk#96], Inner, BuildRight -(118) Project [codegen id : 2] -Output [2]: [ss_quantity#113 AS quantity#117, ss_list_price#114 AS list_price#118] -Input [4]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115, d_date_sk#116] +(106) CometProject +Input [4]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95, d_date_sk#96] +Arguments: [quantity#97, list_price#98], [ss_quantity#93 AS quantity#97, ss_list_price#94 AS list_price#98] -(119) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121] +(107) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#121)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#101)] ReadSchema: struct -(120) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121] +(108) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#102] -(121) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#122] +(109) CometBroadcastHashJoin +Left output [3]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101] +Right output [1]: [d_date_sk#102] +Arguments: [cs_sold_date_sk#101], [d_date_sk#102], Inner, BuildRight -(122) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#121] -Right keys [1]: [d_date_sk#122] -Join type: Inner -Join condition: None +(110) CometProject +Input [4]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101, d_date_sk#102] +Arguments: [quantity#103, list_price#104], [cs_quantity#99 AS quantity#103, cs_list_price#100 AS list_price#104] -(123) Project [codegen id : 4] -Output [2]: [cs_quantity#119 AS quantity#123, cs_list_price#120 AS list_price#124] -Input [4]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121, d_date_sk#122] - -(124) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127] +(111) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#127)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#107)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127] - -(126) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#128] +(112) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#108] -(127) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#127] -Right keys [1]: [d_date_sk#128] -Join type: Inner -Join condition: None +(113) CometBroadcastHashJoin +Left output [3]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107] +Right output [1]: [d_date_sk#108] +Arguments: [ws_sold_date_sk#107], [d_date_sk#108], Inner, BuildRight -(128) Project [codegen id : 6] -Output [2]: [ws_quantity#125 AS quantity#129, ws_list_price#126 AS list_price#130] -Input [4]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127, d_date_sk#128] +(114) CometProject +Input [4]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107, d_date_sk#108] +Arguments: [quantity#109, list_price#110], [ws_quantity#105 AS quantity#109, ws_list_price#106 AS list_price#110] -(129) Union +(115) CometUnion +Child 0 Input [2]: [quantity#97, list_price#98] +Child 1 Input [2]: [quantity#103, list_price#104] +Child 2 Input [2]: [quantity#109, list_price#110] -(130) HashAggregate [codegen id : 7] -Input [2]: [quantity#117, list_price#118] +(116) CometHashAggregate +Input [2]: [quantity#97, list_price#98] Keys: [] -Functions [1]: [partial_avg((cast(quantity#117 as decimal(10,0)) * list_price#118))] -Aggregate Attributes [2]: [sum#131, count#132] -Results [2]: [sum#133, count#134] +Functions [1]: [partial_avg((cast(quantity#97 as decimal(10,0)) * list_price#98))] -(131) Exchange -Input [2]: [sum#133, count#134] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(117) CometExchange +Input [2]: [sum#111, count#112] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(132) HashAggregate [codegen id : 8] -Input [2]: [sum#133, count#134] +(118) CometHashAggregate +Input [2]: [sum#111, count#112] Keys: [] -Functions [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))] -Aggregate Attributes [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))#135] -Results [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))#135 AS average_sales#136] +Functions [1]: [avg((cast(quantity#97 as decimal(10,0)) * list_price#98))] + +(119) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#113] -Subquery:2 Hosting operator id = 90 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] -Subquery:3 Hosting operator id = 106 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/extended.txt new file mode 100644 index 0000000000..302366bdb6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/extended.txt @@ -0,0 +1,410 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometUnion + :- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 403 out of 406 eligible operators (99%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/simplified.txt index 9afe9e29ee..208935d5f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/simplified.txt @@ -1,203 +1,126 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - WholeStageCodegen (80) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - WholeStageCodegen (79) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - InputAdapter - Union - WholeStageCodegen (26) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #1 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #13 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] + CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] + CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #5 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #9 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 - InputAdapter - ReusedExchange [d_date_sk] #9 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (52) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (51) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - InputAdapter - ReusedExchange [d_date_sk] #12 - WholeStageCodegen (78) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #15 - WholeStageCodegen (77) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - InputAdapter - ReusedExchange [d_date_sk] #12 + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #13 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #3 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #5 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #9 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #15 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/explain.txt index bae177c32b..cd5cfdba06 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/explain.txt @@ -1,724 +1,639 @@ == Physical Plan == -TakeOrderedAndProject (96) -+- * BroadcastHashJoin Inner BuildRight (95) - :- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - +- BroadcastExchange (94) - +- * Filter (93) - +- * HashAggregate (92) - +- Exchange (91) - +- * HashAggregate (90) - +- * Project (89) - +- * BroadcastHashJoin Inner BuildRight (88) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : :- * Filter (77) - : : : +- * ColumnarToRow (76) - : : : +- Scan parquet spark_catalog.default.store_sales (75) - : : +- ReusedExchange (78) - : +- ReusedExchange (80) - +- BroadcastExchange (87) - +- * Project (86) - +- * Filter (85) - +- * ColumnarToRow (84) - +- Scan parquet spark_catalog.default.date_dim (83) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [2]: [d_date_sk#39, d_week_seq#40] Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#43, isEmpty#44, count#45] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49, count(1)#50] -Results [6]: [store AS channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49 AS sales#52, count(1)#50 AS number_sales#53] -(74) Filter [codegen id : 52] -Input [6]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53] -Condition : (isnotnull(sales#52) AND (cast(sales#52 as decimal(32,6)) > cast(Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] +(65) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#59)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] - -(77) Filter [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) -(78) ReusedExchange [Reuses operator id: 54] +(67) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(79) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#51], [ss_item_sk#34], LeftSemi, BuildRight -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] -(81) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [i_item_sk#60] -Join type: Inner -Join condition: None +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_item_sk#51], [i_item_sk#55], Inner, BuildRight -(82) Project [codegen id : 50] -Output [6]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [8]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] -(83) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#59, d_week_seq#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(85) Filter [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(86) Project [codegen id : 49] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(87) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(88) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_sold_date_sk#59] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 50] -Output [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [7]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] - -(90) HashAggregate [codegen id : 50] -Input [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), partial_count(1)] -Aggregate Attributes [3]: [sum#68, isEmpty#69, count#70] -Results [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] - -(91) Exchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(92) HashAggregate [codegen id : 51] -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74, count(1)#75] -Results [6]: [store AS channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74 AS sales#77, count(1)#75 AS number_sales#78] - -(93) Filter [codegen id : 51] -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(94) BroadcastExchange -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 52] -Left keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Right keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Join type: Inner -Join condition: None - -(96) TakeOrderedAndProject -Input [12]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] +(73) CometFilter +Input [2]: [d_date_sk#59, d_week_seq#60] +Condition : ((isnotnull(d_week_seq#60) AND (d_week_seq#60 = Subquery scalar-subquery#61, [id=#62])) AND isnotnull(d_date_sk#59)) + +(74) CometProject +Input [2]: [d_date_sk#59, d_week_seq#60] +Arguments: [d_date_sk#59], [d_date_sk#59] + +(75) CometBroadcastExchange +Input [1]: [d_date_sk#59] +Arguments: [d_date_sk#59] + +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [ss_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight + +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] + +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] + +(79) CometExchange +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(80) CometHashAggregate +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] + +(81) CometFilter +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Condition : (isnotnull(sales#67) AND (cast(sales#67 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(82) CometBroadcastExchange +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#56, i_class_id#57, i_category_id#58], Inner, BuildRight + +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#66,i_brand_id#56,i_class_id#57,i_category_id#58,sales#67,number_sales#68]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#54, [id=#55] -* HashAggregate (115) -+- Exchange (114) - +- * HashAggregate (113) - +- Union (112) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * ColumnarToRow (98) - : : +- Scan parquet spark_catalog.default.store_sales (97) - : +- ReusedExchange (99) - :- * Project (106) - : +- * BroadcastHashJoin Inner BuildRight (105) - : :- * ColumnarToRow (103) - : : +- Scan parquet spark_catalog.default.catalog_sales (102) - : +- ReusedExchange (104) - +- * Project (111) - +- * BroadcastHashJoin Inner BuildRight (110) - :- * ColumnarToRow (108) - : +- Scan parquet spark_catalog.default.web_sales (107) - +- ReusedExchange (109) - - -(97) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometNativeScan parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometNativeScan parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#81)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#71)] ReadSchema: struct -(98) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +(87) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#72] -(99) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#82] +(88) CometBroadcastHashJoin +Left output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] +Right output [1]: [d_date_sk#72] +Arguments: [ss_sold_date_sk#71], [d_date_sk#72], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#81] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None +(89) CometProject +Input [4]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71, d_date_sk#72] +Arguments: [quantity#73, list_price#74], [ss_quantity#69 AS quantity#73, ss_list_price#70 AS list_price#74] -(101) Project [codegen id : 2] -Output [2]: [ss_quantity#79 AS quantity#83, ss_list_price#80 AS list_price#84] -Input [4]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81, d_date_sk#82] - -(102) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] +(90) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#87)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] - -(104) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#88] +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#78] -(105) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#87] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] +Right output [1]: [d_date_sk#78] +Arguments: [cs_sold_date_sk#77], [d_date_sk#78], Inner, BuildRight -(106) Project [codegen id : 4] -Output [2]: [cs_quantity#85 AS quantity#89, cs_list_price#86 AS list_price#90] -Input [4]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87, d_date_sk#88] +(93) CometProject +Input [4]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#78] +Arguments: [quantity#79, list_price#80], [cs_quantity#75 AS quantity#79, cs_list_price#76 AS list_price#80] -(107) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(94) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#93)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#83)] ReadSchema: struct -(108) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#84] -(109) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#94] +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] +Right output [1]: [d_date_sk#84] +Arguments: [ws_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight -(110) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#93] -Right keys [1]: [d_date_sk#94] -Join type: Inner -Join condition: None +(97) CometProject +Input [4]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83, d_date_sk#84] +Arguments: [quantity#85, list_price#86], [ws_quantity#81 AS quantity#85, ws_list_price#82 AS list_price#86] -(111) Project [codegen id : 6] -Output [2]: [ws_quantity#91 AS quantity#95, ws_list_price#92 AS list_price#96] -Input [4]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93, d_date_sk#94] +(98) CometUnion +Child 0 Input [2]: [quantity#73, list_price#74] +Child 1 Input [2]: [quantity#79, list_price#80] +Child 2 Input [2]: [quantity#85, list_price#86] -(112) Union - -(113) HashAggregate [codegen id : 7] -Input [2]: [quantity#83, list_price#84] +(99) CometHashAggregate +Input [2]: [quantity#73, list_price#74] Keys: [] -Functions [1]: [partial_avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [2]: [sum#97, count#98] -Results [2]: [sum#99, count#100] +Functions [1]: [partial_avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] -(114) Exchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(100) CometExchange +Input [2]: [sum#87, count#88] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(115) HashAggregate [codegen id : 8] -Input [2]: [sum#99, count#100] +(101) CometHashAggregate +Input [2]: [sum#87, count#88] Keys: [] -Functions [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101] -Results [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101 AS average_sales#102] +Functions [1]: [avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] + +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#89] -Subquery:2 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* Project (119) -+- * Filter (118) - +- * ColumnarToRow (117) - +- Scan parquet spark_catalog.default.date_dim (116) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(116) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(103) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(117) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(104) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 2000)) AND (d_moy#92 = 12)) AND (d_dom#93 = 11)) -(118) Filter [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Condition : (((((isnotnull(d_year#104) AND isnotnull(d_moy#105)) AND isnotnull(d_dom#106)) AND (d_year#104 = 2000)) AND (d_moy#105 = 12)) AND (d_dom#106 = 11)) +(105) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(119) Project [codegen id : 1] -Output [1]: [d_week_seq#103] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] -Subquery:3 Hosting operator id = 93 Hosting Expression = ReusedSubquery Subquery scalar-subquery#54, [id=#55] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:4 Hosting operator id = 85 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* Project (123) -+- * Filter (122) - +- * ColumnarToRow (121) - +- Scan parquet spark_catalog.default.date_dim (120) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#61, [id=#62] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (107) -(120) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(107) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(121) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(108) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1999)) AND (d_moy#92 = 12)) AND (d_dom#93 = 11)) -(122) Filter [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Condition : (((((isnotnull(d_year#108) AND isnotnull(d_moy#109)) AND isnotnull(d_dom#110)) AND (d_year#108 = 1999)) AND (d_moy#109 = 12)) AND (d_dom#110 = 11)) +(109) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(123) Project [codegen id : 1] -Output [1]: [d_week_seq#107] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/extended.txt new file mode 100644 index 0000000000..fc08251119 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/extended.txt @@ -0,0 +1,290 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometBroadcastHashJoin + :- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : +- Subquery + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 280 out of 284 eligible operators (98%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/simplified.txt index b9a845cb11..92429c0948 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/simplified.txt @@ -1,191 +1,122 @@ -TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - WholeStageCodegen (52) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #2 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #1 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #2 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #3 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #4 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #1 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #4 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (51) - Filter [sales] + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (50) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (49) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #3 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/explain.txt index 397cbe07ce..40c3fe1b9e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/explain.txt @@ -1,154 +1,140 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (15) - : +- * BroadcastHashJoin Inner BuildRight (14) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.customer (4) - : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.customer_address (10) - +- BroadcastExchange (20) - +- * Project (19) - +- * Filter (18) - +- * ColumnarToRow (17) - +- Scan parquet spark_catalog.default.date_dim (16) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.customer (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + +- CometBroadcastExchange (17) + +- CometProject (16) + +- CometFilter (15) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(4) Scan parquet spark_catalog.default.customer +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [c_customer_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#4] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight -(9) Project [codegen id : 4] -Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +(7) CometProject Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Condition : isnotnull(ca_address_sk#6) -(13) BroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#7, 2, true, false, true) AS ca_state#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#8, 10, true, false, true) AS ca_zip#10] + +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#6] -Join type: Inner -Join condition: ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)) +(12) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight -(15) Project [codegen id : 4] -Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] +(13) CometProject +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] +(15) CometFilter +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) +(16) CometProject +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] -(20) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(19) CometProject +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] -(22) Project [codegen id : 4] -Output [2]: [cs_sales_price#2, ca_zip#8] -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] - -(23) HashAggregate [codegen id : 4] -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] +(20) CometHashAggregate +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [ca_zip#8, sum#13] -(24) Exchange -Input [2]: [ca_zip#8, sum#13] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 5] -Input [2]: [ca_zip#8, sum#13] -Keys [1]: [ca_zip#8] +(22) CometHashAggregate +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#14] -Results [2]: [ca_zip#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#14,17,2) AS sum(cs_sales_price)#15] -(26) TakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#15] -Arguments: 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#15] +(23) CometTakeOrderedAndProject +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#10 ASC NULLS FIRST], output=[ca_zip#10,sum(cs_sales_price)#15]), [ca_zip#10, sum(cs_sales_price)#15], 100, 0, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#15] + +(24) CometColumnarToRow [codegen id : 1] +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/extended.txt new file mode 100644 index 0000000000..91516fb7af --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/extended.txt @@ -0,0 +1,26 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 23 out of 23 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/simplified.txt index 1509f18b9d..ba08c3f0c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/simplified.txt @@ -1,39 +1,26 @@ -TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - WholeStageCodegen (5) - HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum] - InputAdapter - Exchange [ca_zip] #1 - WholeStageCodegen (4) - HashAggregate [ca_zip,cs_sales_price] [sum,sum] - Project [cs_sales_price,ca_zip] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sales_price,cs_sold_date_sk,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_zip,ca_state,cs_sales_price] - Project [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ca_zip,sum(cs_sales_price)] + CometHashAggregate [sum] [ca_zip,sum(cs_sales_price),sum(UnscaledValue(cs_sales_price))] + CometExchange [ca_zip] #1 + CometHashAggregate [cs_sales_price] [ca_zip,sum] + CometProject [cs_sales_price,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] + CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] + CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] + CometFilter [ca_address_sk,ca_state,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/explain.txt index 70a5c5bed4..2a3c8932ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/explain.txt @@ -1,260 +1,235 @@ == Physical Plan == -* HashAggregate (45) -+- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- * Sort (11) - : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (7) - : : : +- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.catalog_returns (14) - : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) - : : +- Scan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.call_center (34) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometNativeScan parquet spark_catalog.default.call_center (29) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3)) -(4) Project [codegen id : 1] -Output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(3) CometProject Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(5) Exchange +(4) CometExchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: [cs_order_number#5 ASC NULLS FIRST], false, 0 +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.catalog_sales +(6) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] - -(9) Project [codegen id : 3] -Output [2]: [cs_warehouse_sk#9, cs_order_number#10] +(7) CometProject Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] -(10) Exchange +(8) CometExchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: [cs_order_number#10 ASC NULLS FIRST], false, 0 +Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_order_number#10 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 5] -Left keys [1]: [cs_order_number#5] -Right keys [1]: [cs_order_number#10] -Join type: LeftSemi -Join condition: NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) +(10) CometSortMergeJoin +Left output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [2]: [cs_warehouse_sk#9, cs_order_number#10] +Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) -(13) Project [codegen id : 5] -Output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(11) CometProject Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(14) Scan parquet spark_catalog.default.catalog_returns +(12) CometNativeScan parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] ReadSchema: struct -(15) ColumnarToRow [codegen id : 6] +(13) CometProject Input [2]: [cr_order_number#12, cr_returned_date_sk#13] +Arguments: [cr_order_number#12], [cr_order_number#12] -(16) Project [codegen id : 6] -Output [1]: [cr_order_number#12] -Input [2]: [cr_order_number#12, cr_returned_date_sk#13] - -(17) Exchange +(14) CometExchange Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(18) Sort [codegen id : 7] +(15) CometSort Input [1]: [cr_order_number#12] -Arguments: [cr_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [cr_order_number#12], [cr_order_number#12 ASC NULLS FIRST] -(19) SortMergeJoin [codegen id : 11] -Left keys [1]: [cs_order_number#5] -Right keys [1]: [cr_order_number#12] -Join type: LeftAnti -Join condition: None +(16) CometSortMergeJoin +Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [cr_order_number#12] +Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(20) Scan parquet spark_catalog.default.date_dim +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] +(18) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2002-02-01)) AND (d_date#15 <= 2002-04-02)) AND isnotnull(d_date_sk#14)) -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] +(19) CometProject Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] -(24) BroadcastExchange +(20) CometBroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#14] -(25) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [d_date_sk#14] +Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight -(26) Project [codegen id : 11] -Output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(22) CometProject Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] +Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(27) Scan parquet spark_catalog.default.customer_address +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] +(24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#16)) -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] +(25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] -(31) BroadcastExchange +(26) CometBroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#16] -(32) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [ca_address_sk#16] +Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight -(33) Project [codegen id : 11] -Output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(28) CometProject Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] +Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(34) Scan parquet spark_catalog.default.call_center +(29) CometNativeScan parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [cc_call_center_sk#18, cc_county#19] - -(36) Filter [codegen id : 10] +(30) CometFilter Input [2]: [cc_call_center_sk#18, cc_county#19] Condition : ((isnotnull(cc_county#19) AND (cc_county#19 = Williamson County)) AND isnotnull(cc_call_center_sk#18)) -(37) Project [codegen id : 10] -Output [1]: [cc_call_center_sk#18] +(31) CometProject Input [2]: [cc_call_center_sk#18, cc_county#19] +Arguments: [cc_call_center_sk#18], [cc_call_center_sk#18] -(38) BroadcastExchange +(32) CometBroadcastExchange Input [1]: [cc_call_center_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [cc_call_center_sk#18] -(39) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_call_center_sk#3] -Right keys [1]: [cc_call_center_sk#18] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [cc_call_center_sk#18] +Arguments: [cs_call_center_sk#3], [cc_call_center_sk#18], Inner, BuildRight -(40) Project [codegen id : 11] -Output [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(34) CometProject Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#18] +Arguments: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(41) HashAggregate [codegen id : 11] +(35) CometHashAggregate Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Keys [1]: [cs_order_number#5] Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_order_number#5, sum#22, sum#23] +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] + +(37) HashAggregate [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] Keys [1]: [cs_order_number#5] Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23] +Results [3]: [cs_order_number#5, sum#20, sum#21] -(43) HashAggregate [codegen id : 11] -Input [3]: [cs_order_number#5, sum#22, sum#23] +(38) HashAggregate [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] Keys: [] Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] -(44) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(45) HashAggregate [codegen id : 12] -Input [3]: [sum#22, sum#23, count#25] +(40) CometHashAggregate +Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#21,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/extended.txt new file mode 100644 index 0000000000..9889eeaca3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/simplified.txt index 0ae1584bd9..429d83d08c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/simplified.txt @@ -1,74 +1,45 @@ -WholeStageCodegen (12) - HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] +WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] - Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - SortMergeJoin [cs_order_number,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - Sort [cs_order_number] - InputAdapter - Exchange [cs_order_number] #2 - WholeStageCodegen (1) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - Filter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number] - InputAdapter - Exchange [cs_order_number] #3 - WholeStageCodegen (3) - Project [cs_warehouse_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_order_number] - InputAdapter - Exchange [cr_order_number] #4 - WholeStageCodegen (6) - Project [cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Project [cc_call_center_sk] - Filter [cc_county,cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(cs_order_number),sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [cs_ext_ship_cost,cs_net_profit] [cs_order_number,sum,sum] + CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] + CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] + CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cc_call_center_sk] #7 + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/explain.txt index f5b068a455..0e4ffc8864 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/explain.txt @@ -1,279 +1,254 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- ReusedExchange (30) - : +- BroadcastExchange (36) - : +- * Filter (35) - : +- * ColumnarToRow (34) - : +- Scan parquet spark_catalog.default.store (33) - +- BroadcastExchange (42) - +- * Filter (41) - +- * ColumnarToRow (40) - +- Scan parquet spark_catalog.default.item (39) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.item (34) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +(7) CometProject Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(10) Scan parquet spark_catalog.default.catalog_sales +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(12) CometProject Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_quarter_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#16, d_quarter_name#17] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#17, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#16)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] +(15) CometProject Input [2]: [d_date_sk#16, d_quarter_name#17] +Arguments: [d_date_sk#16], [d_date_sk#16] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#16] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(18) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(23) Scan parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_quarter_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#18, d_quarter_name#19] - -(25) Filter [codegen id : 4] +(20) CometFilter Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#19, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#18] +(21) CometProject Input [2]: [d_date_sk#18, d_quarter_name#19] +Arguments: [d_date_sk#18], [d_date_sk#18] -(27) BroadcastExchange +(22) CometBroadcastExchange Input [1]: [d_date_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#18] -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +(24) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -(30) ReusedExchange [Reuses operator id: 27] +(25) ReusedExchange [Reuses operator id: 22] Output [1]: [d_date_sk#20] -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight -(32) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +(27) CometProject Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(33) Scan parquet spark_catalog.default.store +(28) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#21, s_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#21, s_state#22] - -(35) Filter [codegen id : 6] +(29) CometFilter Input [2]: [s_store_sk#21, s_state#22] Condition : isnotnull(s_store_sk#21) -(36) BroadcastExchange +(30) CometProject Input [2]: [s_store_sk#21, s_state#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#22, 2, true, false, true) AS s_state#23] + +(31) CometBroadcastExchange +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None +(32) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [2]: [s_store_sk#21, s_state#23] +Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight -(38) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] +(33) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] -(39) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(34) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(35) CometFilter +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) -(41) Filter [codegen id : 7] -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) +(36) CometProject +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#25, 16, true, false, true) AS i_item_id#27, i_item_desc#26] -(42) BroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(37) CometBroadcastExchange +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#23] -Join type: Inner -Join condition: None +(38) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight -(44) Project [codegen id : 8] -Output [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] +(39) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] -(45) HashAggregate [codegen id : 8] -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(40) CometHashAggregate +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [18]: [count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Results [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -(46) Exchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometExchange +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(47) HashAggregate [codegen id : 9] -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(42) CometHashAggregate +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [9]: [count(ss_quantity#5)#62, avg(ss_quantity#5)#63, stddev_samp(cast(ss_quantity#5 as double))#64, count(sr_return_quantity#10)#65, avg(sr_return_quantity#10)#66, stddev_samp(cast(sr_return_quantity#10 as double))#67, count(cs_quantity#14)#68, avg(cs_quantity#14)#69, stddev_samp(cast(cs_quantity#14 as double))#70] -Results [15]: [i_item_id#24, i_item_desc#25, s_state#22, count(ss_quantity#5)#62 AS store_sales_quantitycount#71, avg(ss_quantity#5)#63 AS store_sales_quantityave#72, stddev_samp(cast(ss_quantity#5 as double))#64 AS store_sales_quantitystdev#73, (stddev_samp(cast(ss_quantity#5 as double))#64 / avg(ss_quantity#5)#63) AS store_sales_quantitycov#74, count(sr_return_quantity#10)#65 AS as_store_returns_quantitycount#75, avg(sr_return_quantity#10)#66 AS as_store_returns_quantityave#76, stddev_samp(cast(sr_return_quantity#10 as double))#67 AS as_store_returns_quantitystdev#77, (stddev_samp(cast(sr_return_quantity#10 as double))#67 / avg(sr_return_quantity#10)#66) AS store_returns_quantitycov#78, count(cs_quantity#14)#68 AS catalog_sales_quantitycount#79, avg(cs_quantity#14)#69 AS catalog_sales_quantityave#80, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitystdev#81, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitycov#82] -(48) TakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] -Arguments: 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] +(43) CometTakeOrderedAndProject +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] + +(44) CometColumnarToRow [codegen id : 1] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/simplified.txt index a951b39350..97c37ed5a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/simplified.txt @@ -1,71 +1,46 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - InputAdapter - Exchange [i_item_id,i_item_desc,s_state] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - Project [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] + CometHashAggregate [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] + CometExchange [i_item_id,i_item_desc,s_state] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_state] #6 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/explain.txt index 6e2c40bca4..c33d4125b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/explain.txt @@ -1,271 +1,247 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Expand (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (23) - : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.customer (11) - : : : +- BroadcastExchange (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.customer_demographics (18) - : : +- BroadcastExchange (27) - : : +- * Filter (26) - : : +- * ColumnarToRow (25) - : : +- Scan parquet spark_catalog.default.customer_address (24) - : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.date_dim (30) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.item (37) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (43) ++- CometTakeOrderedAndProject (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometExpand (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.date_dim (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.item (32) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(3) Filter [codegen id : 7] +(2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_education_status,Unknown ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#10)) -(6) Filter [codegen id : 1] +(5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#10, cd_dep_count#13] -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight -(10) Project [codegen id : 7] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +(8) CometProject Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(11) Scan parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(11) CometProject Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight -(17) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(14) CometProject Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(18) Scan parquet spark_catalog.default.customer_demographics +(15) CometNativeScan parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [cd_demo_sk#19] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [cd_demo_sk#19] Condition : isnotnull(cd_demo_sk#19) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [cd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [cd_demo_sk#19] -(22) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight -(23) Project [codegen id : 7] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +(19) CometProject Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(24) Scan parquet spark_catalog.default.customer_address +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [IN,MS,ND,NM,OK,VA]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) -(26) Filter [codegen id : 4] +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] -(27) BroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(29) Project [codegen id : 7] -Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(30) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +(26) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] - -(32) Filter [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#24] -Input [2]: [d_date_sk#24, d_year#25] +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] -(34) BroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight -(36) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(37) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) -(39) Filter [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] -(40) BroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight -(42) Project [codegen id : 7] -Output [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] -(43) Expand [codegen id : 7] -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(38) CometExpand +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] -(44) HashAggregate [codegen id : 7] -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(39) CometHashAggregate +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [14]: [sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Results [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -(45) Exchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(40) CometExchange +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(46) HashAggregate [codegen id : 8] -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(41) CometHashAggregate +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#61, avg(cast(cs_list_price#5 as decimal(12,2)))#62, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63, avg(cast(cs_sales_price#6 as decimal(12,2)))#64, avg(cast(cs_net_profit#8 as decimal(12,2)))#65, avg(cast(c_birth_year#18 as decimal(12,2)))#66, avg(cast(cd_dep_count#13 as decimal(12,2)))#67] -Results [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, avg(cast(cs_quantity#4 as decimal(12,2)))#61 AS agg1#68, avg(cast(cs_list_price#5 as decimal(12,2)))#62 AS agg2#69, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63 AS agg3#70, avg(cast(cs_sales_price#6 as decimal(12,2)))#64 AS agg4#71, avg(cast(cs_net_profit#8 as decimal(12,2)))#65 AS agg5#72, avg(cast(c_birth_year#18 as decimal(12,2)))#66 AS agg6#73, avg(cast(cd_dep_count#13 as decimal(12,2)))#67 AS agg7#74] -(47) TakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] -Arguments: 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] +(42) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] + +(43) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/extended.txt new file mode 100644 index 0000000000..63cf56de78 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/simplified.txt index bd9fc33e49..4507330941 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/simplified.txt @@ -1,69 +1,45 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - WholeStageCodegen (8) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Expand [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] + CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/explain.txt index 2ee7d276e1..78e43d0122 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/explain.txt @@ -1,227 +1,208 @@ == Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (17) - : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.date_dim (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) - : : : : +- Scan parquet spark_catalog.default.store_sales (5) - : : : +- BroadcastExchange (15) - : : : +- * Project (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.item (11) - : : +- BroadcastExchange (21) - : : +- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.customer (18) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.customer_address (24) - +- BroadcastExchange (33) - +- * Filter (32) - +- * ColumnarToRow (31) - +- Scan parquet spark_catalog.default.store (30) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (36) ++- CometTakeOrderedAndProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.item (9) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.customer (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.store (26) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 6] -Output [1]: [d_date_sk#1] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[4, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#8] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [1]: [d_date_sk#1] +Right output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight -(10) Project [codegen id : 6] -Output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] +(8) CometProject Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) -(14) Project [codegen id : 2] -Output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +(11) CometProject Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [i_item_sk#9, i_brand_id#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#11, 50, true, false, true) AS i_brand#15, i_manufact_id#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#13, 50, true, false, true) AS i_manufact#16] -(15) BroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(16) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] +Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight -(17) Project [codegen id : 6] -Output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +(14) CometProject +Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(18) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] +(15) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#17, c_current_addr_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] +(16) CometFilter +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) -(20) Filter [codegen id : 3] -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) +(17) CometBroadcastExchange +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [c_customer_sk#17, c_current_addr_sk#18] -(21) BroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#5] -Right keys [1]: [c_customer_sk#15] -Join type: Inner -Join condition: None +(19) CometProject +Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] -(23) Project [codegen id : 6] -Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] - -(24) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_zip#18] +(20) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#19, ca_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_zip#18] +(21) CometFilter +Input [2]: [ca_address_sk#19, ca_zip#20] +Condition : (isnotnull(ca_address_sk#19) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true))) -(26) Filter [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) +(22) CometProject +Input [2]: [ca_address_sk#19, ca_zip#20] +Arguments: [ca_address_sk#19, ca_zip#21], [ca_address_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true) AS ca_zip#21] -(27) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [ca_address_sk#19, ca_zip#21] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] +Right output [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight -(29) Project [codegen id : 6] -Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] +(25) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] -(30) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#19, s_zip#20] +(26) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#22, s_zip#23] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [s_store_sk#19, s_zip#20] +(27) CometFilter +Input [2]: [s_store_sk#22, s_zip#23] +Condition : (isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true)) AND isnotnull(s_store_sk#22)) -(32) Filter [codegen id : 5] -Input [2]: [s_store_sk#19, s_zip#20] -Condition : (isnotnull(s_zip#20) AND isnotnull(s_store_sk#19)) +(28) CometProject +Input [2]: [s_store_sk#22, s_zip#23] +Arguments: [s_store_sk#22, s_zip#24], [s_store_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true) AS s_zip#24] -(33) BroadcastExchange -Input [2]: [s_store_sk#19, s_zip#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [2]: [s_store_sk#22, s_zip#24] +Arguments: [s_store_sk#22, s_zip#24] -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#19] -Join type: Inner -Join condition: NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)) +(30) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] +Right output [2]: [s_store_sk#22, s_zip#24] +Arguments: [ss_store_sk#6], [s_store_sk#22], Inner, NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5)), BuildRight -(35) Project [codegen id : 6] -Output [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] +(31) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21, s_store_sk#22, s_zip#24] +Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(36) HashAggregate [codegen id : 6] -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(32) CometHashAggregate +Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum#21] -Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -(37) Exchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometExchange +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Arguments: hashpartitioning(i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(38) HashAggregate [codegen id : 7] -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(34) CometHashAggregate +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#23] -Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#23,17,2) AS ext_price#26] -(39) TakeOrderedAndProject -Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] -Arguments: 100, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] +(35) CometTakeOrderedAndProject +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#28 DESC NULLS LAST,brand#27 ASC NULLS FIRST,brand_id#26 ASC NULLS FIRST,i_manufact_id#12 ASC NULLS FIRST,i_manufact#16 ASC NULLS FIRST], output=[brand_id#26,brand#27,i_manufact_id#12,i_manufact#16,ext_price#28]), [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28], 100, 0, [ext_price#28 DESC NULLS LAST, brand#27 ASC NULLS FIRST, brand_id#26 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#16 ASC NULLS FIRST], [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] + +(36) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/extended.txt new file mode 100644 index 0000000000..fbe6798e3d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/extended.txt @@ -0,0 +1,38 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 35 out of 35 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/simplified.txt index 65d7e34ffe..675500cd0b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/simplified.txt @@ -1,58 +1,38 @@ -TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] - WholeStageCodegen (7) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - WholeStageCodegen (6) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - BroadcastHashJoin [ss_store_sk,s_store_sk,ca_zip,s_zip] - Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [s_zip,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_zip] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [brand_id,brand,i_manufact_id,i_manufact,ext_price] + CometHashAggregate [sum] [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] + CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 + CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_zip] #5 + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [s_store_sk,s_zip] #6 + CometProject [s_zip] [s_store_sk,s_zip] + CometFilter [s_store_sk,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/explain.txt index 72bee887d4..22fbab2817 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/explain.txt @@ -1,212 +1,193 @@ == Physical Plan == -* Sort (37) -+- Exchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * HashAggregate (16) - : : +- Exchange (15) - : : +- * HashAggregate (14) - : : +- * Project (13) - : : +- * BroadcastHashJoin Inner BuildRight (12) - : : :- Union (7) - : : : :- * Project (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- * Project (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : +- BroadcastExchange (11) - : : +- * Filter (10) - : : +- * ColumnarToRow (9) - : : +- Scan parquet spark_catalog.default.date_dim (8) - : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.date_dim (17) - +- BroadcastExchange (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * HashAggregate (25) - : +- ReusedExchange (24) - +- BroadcastExchange (30) - +- * Project (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.date_dim (26) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (34) ++- CometSort (33) + +- CometExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometHashAggregate (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) + : : :- CometUnion (5) + : : : :- CometProject (2) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : +- CometProject (4) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (3) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (6) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.date_dim (15) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometHashAggregate (22) + : +- ReusedExchange (21) + +- CometBroadcastExchange (26) + +- CometProject (25) + +- CometFilter (24) + +- CometNativeScan parquet spark_catalog.default.date_dim (23) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#2)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometProject Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] +Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) Project [codegen id : 1] -Output [2]: [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] - -(4) Scan parquet spark_catalog.default.catalog_sales +(3) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#6)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] - -(6) Project [codegen id : 2] -Output [2]: [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] +(4) CometProject Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] +Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] -(7) Union +(5) CometUnion +Child 0 Input [2]: [sold_date_sk#3, sales_price#4] +Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(8) Scan parquet spark_catalog.default.date_dim +(6) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(9) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] - -(10) Filter [codegen id : 3] +(7) CometFilter Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(11) BroadcastExchange +(8) CometProject Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12], [d_date_sk#9, d_week_seq#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#11, 9, true, false, true) AS d_day_name#12] -(12) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [sold_date_sk#3] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(9) CometBroadcastExchange +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12] -(13) Project [codegen id : 4] -Output [3]: [sales_price#4, d_week_seq#10, d_day_name#11] -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] +(10) CometBroadcastHashJoin +Left output [2]: [sold_date_sk#3, sales_price#4] +Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(14) HashAggregate [codegen id : 4] -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] +(11) CometProject +Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sales_price#4, d_week_seq#10, d_day_name#12], [sales_price#4, d_week_seq#10, d_day_name#12] + +(12) CometHashAggregate +Input [3]: [sales_price#4, d_week_seq#10, d_day_name#12] Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(15) Exchange -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(13) CometExchange +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) HashAggregate [codegen id : 12] -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] +(14) CometHashAggregate +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_week_seq#40, d_year#41] +(15) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 5] -Input [2]: [d_week_seq#40, d_year#41] - -(19) Filter [codegen id : 5] -Input [2]: [d_week_seq#40, d_year#41] -Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2001)) AND isnotnull(d_week_seq#40)) +(16) CometFilter +Input [2]: [d_week_seq#20, d_year#21] +Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_week_seq#20)) -(20) Project [codegen id : 5] -Output [1]: [d_week_seq#40] -Input [2]: [d_week_seq#40, d_year#41] +(17) CometProject +Input [2]: [d_week_seq#20, d_year#21] +Arguments: [d_week_seq#20], [d_week_seq#20] -(21) BroadcastExchange -Input [1]: [d_week_seq#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [1]: [d_week_seq#20] +Arguments: [d_week_seq#20] -(22) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#40] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [8]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28] +Right output [1]: [d_week_seq#20] +Arguments: [d_week_seq#10], [d_week_seq#20], Inner, BuildRight -(23) Project [codegen id : 12] -Output [8]: [d_week_seq#10 AS d_week_seq1#42, sun_sales#33 AS sun_sales1#43, mon_sales#34 AS mon_sales1#44, tue_sales#35 AS tue_sales1#45, wed_sales#36 AS wed_sales1#46, thu_sales#37 AS thu_sales1#47, fri_sales#38 AS fri_sales1#48, sat_sales#39 AS sat_sales1#49] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#40] +(20) CometProject +Input [9]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, d_week_seq#20] +Arguments: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36], [d_week_seq#10 AS d_week_seq1#29, sun_sales#22 AS sun_sales1#30, mon_sales#23 AS mon_sales1#31, tue_sales#24 AS tue_sales1#32, wed_sales#25 AS wed_sales1#33, thu_sales#26 AS thu_sales1#34, fri_sales#27 AS fri_sales1#35, sat_sales#28 AS sat_sales1#36] -(24) ReusedExchange [Reuses operator id: 15] -Output [8]: [d_week_seq#10, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(21) ReusedExchange [Reuses operator id: 13] +Output [8]: [d_week_seq#10, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] -(25) HashAggregate [codegen id : 11] -Input [8]: [d_week_seq#10, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(22) CometHashAggregate +Input [8]: [d_week_seq#10, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_week_seq#57, d_year#58] +(23) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 10] -Input [2]: [d_week_seq#57, d_year#58] +(24) CometFilter +Input [2]: [d_week_seq#44, d_year#45] +Condition : ((isnotnull(d_year#45) AND (d_year#45 = 2002)) AND isnotnull(d_week_seq#44)) -(28) Filter [codegen id : 10] -Input [2]: [d_week_seq#57, d_year#58] -Condition : ((isnotnull(d_year#58) AND (d_year#58 = 2002)) AND isnotnull(d_week_seq#57)) +(25) CometProject +Input [2]: [d_week_seq#44, d_year#45] +Arguments: [d_week_seq#44], [d_week_seq#44] -(29) Project [codegen id : 10] -Output [1]: [d_week_seq#57] -Input [2]: [d_week_seq#57, d_year#58] +(26) CometBroadcastExchange +Input [1]: [d_week_seq#44] +Arguments: [d_week_seq#44] -(30) BroadcastExchange -Input [1]: [d_week_seq#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(27) CometBroadcastHashJoin +Left output [8]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28] +Right output [1]: [d_week_seq#44] +Arguments: [d_week_seq#10], [d_week_seq#44], Inner, BuildRight -(31) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#57] -Join type: Inner -Join condition: None +(28) CometProject +Input [9]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, d_week_seq#44] +Arguments: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53], [d_week_seq#10 AS d_week_seq2#46, sun_sales#22 AS sun_sales2#47, mon_sales#23 AS mon_sales2#48, tue_sales#24 AS tue_sales2#49, wed_sales#25 AS wed_sales2#50, thu_sales#26 AS thu_sales2#51, fri_sales#27 AS fri_sales2#52, sat_sales#28 AS sat_sales2#53] -(32) Project [codegen id : 11] -Output [8]: [d_week_seq#10 AS d_week_seq2#59, sun_sales#33 AS sun_sales2#60, mon_sales#34 AS mon_sales2#61, tue_sales#35 AS tue_sales2#62, wed_sales#36 AS wed_sales2#63, thu_sales#37 AS thu_sales2#64, fri_sales#38 AS fri_sales2#65, sat_sales#39 AS sat_sales2#66] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#57] +(29) CometBroadcastExchange +Input [8]: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] +Arguments: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] -(33) BroadcastExchange -Input [8]: [d_week_seq2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=5] +(30) CometBroadcastHashJoin +Left output [8]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36] +Right output [8]: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] +Arguments: [d_week_seq1#29], [(d_week_seq2#46 - 53)], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [d_week_seq1#42] -Right keys [1]: [(d_week_seq2#59 - 53)] -Join type: Inner -Join condition: None +(31) CometProject +Input [16]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36, d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60], [d_week_seq1#29, round((sun_sales1#30 / sun_sales2#47), 2) AS round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1#31 / mon_sales2#48), 2) AS round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1#32 / tue_sales2#49), 2) AS round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1#33 / wed_sales2#50), 2) AS round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1#34 / thu_sales2#51), 2) AS round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1#35 / fri_sales2#52), 2) AS round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1#36 / sat_sales2#53), 2) AS round((sat_sales1 / sat_sales2), 2)#60] -(35) Project [codegen id : 12] -Output [8]: [d_week_seq1#42, round((sun_sales1#43 / sun_sales2#60), 2) AS round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1#44 / mon_sales2#61), 2) AS round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1#45 / tue_sales2#62), 2) AS round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1#46 / wed_sales2#63), 2) AS round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1#47 / thu_sales2#64), 2) AS round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1#48 / fri_sales2#65), 2) AS round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1#49 / sat_sales2#66), 2) AS round((sat_sales1 / sat_sales2), 2)#73] -Input [16]: [d_week_seq1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] +(32) CometExchange +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60] +Arguments: rangepartitioning(d_week_seq1#29 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(36) Exchange -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1 / sat_sales2), 2)#73] -Arguments: rangepartitioning(d_week_seq1#42 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60], [d_week_seq1#29 ASC NULLS FIRST] -(37) Sort [codegen id : 13] -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1 / sat_sales2), 2)#73] -Arguments: [d_week_seq1#42 ASC NULLS FIRST], true, 0 +(34) CometColumnarToRow [codegen id : 1] +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/extended.txt new file mode 100644 index 0000000000..7e6f4d3a3c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/simplified.txt index 1b34ac798b..e4b6e81639 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/simplified.txt @@ -1,59 +1,36 @@ -WholeStageCodegen (13) - Sort [d_week_seq1] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [d_week_seq1] #1 - WholeStageCodegen (12) - Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [d_week_seq1,d_week_seq2] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq] #2 - WholeStageCodegen (4) - HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,d_week_seq,d_day_name] - BroadcastHashJoin [sold_date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ws_sold_date_sk,ws_ext_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] - WholeStageCodegen (2) - Project [cs_sold_date_sk,cs_ext_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (11) - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (10) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] + CometExchange [d_week_seq1] #1 + CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] + CometBroadcastHashJoin [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + CometExchange [d_week_seq] #2 + CometHashAggregate [d_day_name,sales_price] [d_week_seq,sum,sum,sum,sum,sum,sum,sum] + CometProject [sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] + CometUnion [sold_date_sk,sales_price] + CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometBroadcastExchange [d_week_seq] #4 + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 + CometBroadcastExchange [d_week_seq] #6 + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/explain.txt index 96183beea0..684cbba7f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/explain.txt @@ -1,140 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.catalog_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/extended.txt new file mode 100644 index 0000000000..7fd40cf7dd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/simplified.txt index bcc94f0169..0b88804c69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/explain.txt index 7d51ef43a1..68db1e1ee5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/explain.txt @@ -1,159 +1,140 @@ == Physical Plan == -TakeOrderedAndProject (27) -+- * Filter (26) - +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.warehouse (4) - : +- BroadcastExchange (14) - : +- * Project (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.item (10) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.date_dim (17) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometFilter (22) + +- CometHashAggregate (21) + +- CometExchange (20) + +- CometHashAggregate (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometNativeScan parquet spark_catalog.default.item (8) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometNativeScan parquet spark_catalog.default.date_dim (14) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] Condition : isnotnull(w_warehouse_sk#5) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#5, w_warehouse_name#6] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#5] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight -(9) Project [codegen id : 4] -Output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +(7) CometProject Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -(10) Scan parquet spark_catalog.default.item +(8) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) -(13) Project [codegen id : 2] -Output [2]: [i_item_sk#7, i_item_id#8] +(10) CometProject Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#8, 16, true, false, true) AS i_item_id#10] -(14) BroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(11) CometBroadcastExchange +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +Right output [2]: [i_item_sk#7, i_item_id#10] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(16) Project [codegen id : 4] -Output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] +(13) CometProject +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#10, d_date#11] - -(19) Filter [codegen id : 3] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) - -(20) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] - -(23) HashAggregate [codegen id : 4] -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum#12, sum#13] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] - -(24) Exchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16 AS inv_before#18, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17 AS inv_after#19] - -(26) Filter [codegen id : 5] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Condition : (CASE WHEN (inv_before#18 > 0) THEN ((cast(inv_after#19 as double) / cast(inv_before#18 as double)) >= 0.666667) END AND CASE WHEN (inv_before#18 > 0) THEN ((cast(inv_after#19 as double) / cast(inv_before#18 as double)) <= 1.5) END) - -(27) TakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Arguments: 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] +(15) CometFilter +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) + +(16) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] + +(17) CometBroadcastHashJoin +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight + +(18) CometProject +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] + +(19) CometHashAggregate +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(20) CometExchange +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(22) CometFilter +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) + +(23) CometTakeOrderedAndProject +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] + +(24) CometColumnarToRow [codegen id : 1] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/extended.txt new file mode 100644 index 0000000000..92628ea98d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/extended.txt @@ -0,0 +1,26 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 23 out of 23 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/simplified.txt index f09a8ad603..f0cc321417 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/simplified.txt @@ -1,40 +1,26 @@ -TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] - WholeStageCodegen (5) - Filter [inv_before,inv_after] - HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum] - InputAdapter - Exchange [w_warehouse_name,i_item_id] #1 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum] - Project [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Filter [inv_warehouse_sk,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] + CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] + CometHashAggregate [sum,sum] [w_warehouse_name,i_item_id,inv_before,inv_after,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] + CometExchange [w_warehouse_name,i_item_id] #1 + CometHashAggregate [d_date,inv_quantity_on_hand] [w_warehouse_name,i_item_id,sum,sum] + CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] + CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] + CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_id] #3 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #4 + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/explain.txt index 013ede591c..fa4f406442 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/explain.txt @@ -1,159 +1,145 @@ == Physical Plan == -TakeOrderedAndProject (27) -+- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Expand (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.warehouse (17) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (25) ++- CometTakeOrderedAndProject (24) + +- CometHashAggregate (23) + +- CometExchange (22) + +- CometHashAggregate (21) + +- CometExpand (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.warehouse (15) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 4] -Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +(8) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] +Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Condition : isnotnull(i_item_sk#7) -(14) BroadcastExchange +(11) CometProject Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(16) Project [codegen id : 4] -Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(17) Scan parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#12] +(15) CometNativeScan parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [1]: [w_warehouse_sk#12] +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) -(19) Filter [codegen id : 3] -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] -(20) BroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#12] -Join type: Inner -Join condition: None +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] +(20) CometExpand +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] -(23) Expand [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(24) HashAggregate [codegen id : 4] -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(21) CometHashAggregate +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#18, count#19] -Results [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -(25) Exchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(23) CometHashAggregate +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#22] -Results [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, avg(inv_quantity_on_hand#3)#22 AS qoh#23] -(27) TakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] -Arguments: 100, [qoh#23 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] +(24) CometTakeOrderedAndProject +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] + +(25) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/extended.txt new file mode 100644 index 0000000000..275cecfd63 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/simplified.txt index 7346844cad..8d2b671164 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/simplified.txt @@ -1,40 +1,27 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (5) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [spark_grouping_id,sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] + CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] + CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/explain.txt index 886ec7c698..a32c4b9030 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/explain.txt @@ -1,547 +1,484 @@ == Physical Plan == -* HashAggregate (74) -+- Exchange (73) - +- * HashAggregate (72) - +- Union (71) - :- * Project (53) - : +- * BroadcastHashJoin Inner BuildRight (52) - : :- * Project (46) - : : +- * SortMergeJoin LeftSemi (45) - : : :- * Sort (28) - : : : +- Exchange (27) - : : : +- * Project (26) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (25) - : : : :- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (24) - : : : +- * Project (23) - : : : +- * Filter (22) - : : : +- * HashAggregate (21) - : : : +- Exchange (20) - : : : +- * HashAggregate (19) - : : : +- * Project (18) - : : : +- * BroadcastHashJoin Inner BuildRight (17) - : : : :- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * Filter (5) - : : : : : +- * ColumnarToRow (4) - : : : : : +- Scan parquet spark_catalog.default.store_sales (3) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (16) - : : : +- * Filter (15) - : : : +- * ColumnarToRow (14) - : : : +- Scan parquet spark_catalog.default.item (13) - : : +- * Sort (44) - : : +- * Project (43) - : : +- * Filter (42) - : : +- * HashAggregate (41) - : : +- Exchange (40) - : : +- * HashAggregate (39) - : : +- * Project (38) - : : +- * BroadcastHashJoin Inner BuildRight (37) - : : :- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.store_sales (29) - : : +- BroadcastExchange (36) - : : +- * Filter (35) - : : +- * ColumnarToRow (34) - : : +- Scan parquet spark_catalog.default.customer (33) - : +- BroadcastExchange (51) - : +- * Project (50) - : +- * Filter (49) - : +- * ColumnarToRow (48) - : +- Scan parquet spark_catalog.default.date_dim (47) - +- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (67) - : +- * SortMergeJoin LeftSemi (66) - : :- * Sort (60) - : : +- Exchange (59) - : : +- * Project (58) - : : +- * BroadcastHashJoin LeftSemi BuildRight (57) - : : :- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (56) - : +- * Sort (65) - : +- * Project (64) - : +- * Filter (63) - : +- * HashAggregate (62) - : +- ReusedExchange (61) - +- ReusedExchange (68) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (67) ++- CometHashAggregate (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometUnion (63) + :- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (40) + : : +- CometSortMergeJoin (39) + : : :- CometSort (24) + : : : +- CometExchange (23) + : : : +- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (20) + : : : +- CometProject (19) + : : : +- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometFilter (3) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (2) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometNativeScan parquet spark_catalog.default.item (10) + : : +- CometSort (38) + : : +- CometProject (37) + : : +- CometFilter (36) + : : +- CometHashAggregate (35) + : : +- CometExchange (34) + : : +- CometHashAggregate (33) + : : +- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometProject (27) + : : : +- CometFilter (26) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (25) + : : +- CometBroadcastExchange (30) + : : +- CometFilter (29) + : : +- CometNativeScan parquet spark_catalog.default.customer (28) + : +- CometBroadcastExchange (44) + : +- CometProject (43) + : +- CometFilter (42) + : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + +- CometProject (62) + +- CometBroadcastHashJoin (61) + :- CometProject (59) + : +- CometSortMergeJoin (58) + : :- CometSort (52) + : : +- CometExchange (51) + : : +- CometProject (50) + : : +- CometBroadcastHashJoin (49) + : : :- CometNativeScan parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (48) + : +- CometSort (57) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometHashAggregate (54) + : +- ReusedExchange (53) + +- ReusedExchange (60) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#5)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(3) Scan parquet spark_catalog.default.store_sales +(2) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(4) ColumnarToRow [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] - -(5) Filter [codegen id : 3] +(3) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(6) Scan parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [2]: [d_date_sk#8, d_date#9] +(6) CometProject Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [2]: [d_date_sk#8, d_date#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8, d_date#9] -(11) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 3] -Output [2]: [ss_item_sk#6, d_date#9] +(9) CometProject Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(13) Scan parquet spark_catalog.default.item +(10) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] - -(15) Filter [codegen id : 2] +(11) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] Condition : isnotnull(i_item_sk#11) -(16) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#11, i_item_desc#12] -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#6] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight -(18) Project [codegen id : 3] -Output [3]: [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] +(14) CometProject Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] -(19) HashAggregate [codegen id : 3] +(15) CometHashAggregate Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#14] -Results [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -(20) Exchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 4] -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] +(17) CometHashAggregate +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#16] -Results [2]: [i_item_sk#11 AS item_sk#17, count(1)#16 AS cnt#18] -(22) Filter [codegen id : 4] -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +(18) CometFilter +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) -(23) Project [codegen id : 4] -Output [1]: [item_sk#17] -Input [2]: [item_sk#17, cnt#18] +(19) CometProject +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] -(24) BroadcastExchange -Input [1]: [item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(20) CometBroadcastExchange +Input [1]: [item_sk#15] +Arguments: [item_sk#15] -(25) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight -(26) Project [codegen id : 5] -Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +(22) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(27) Exchange +(23) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(28) Sort [codegen id : 6] +(24) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(25) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(31) Filter [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +(26) CometFilter +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) -(32) Project [codegen id : 8] -Output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(27) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(33) Scan parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +(28) CometNativeScan parquet spark_catalog.default.customer +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 7] -Input [1]: [c_customer_sk#23] - -(35) Filter [codegen id : 7] -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) - -(36) BroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#19] -Right keys [1]: [c_customer_sk#23] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 8] -Output [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] - -(39) HashAggregate [codegen id : 8] -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [2]: [sum#24, isEmpty#25] -Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(40) Exchange -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(41) HashAggregate [codegen id : 9] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(42) Filter [codegen id : 9] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) - -(43) Project [codegen id : 9] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(44) Sort [codegen id : 9] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(45) SortMergeJoin [codegen id : 11] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(46) Project [codegen id : 11] -Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +(29) CometFilter +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) + +(30) CometBroadcastExchange +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] + +(31) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight + +(32) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] + +(33) CometHashAggregate +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(34) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(35) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(36) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) + +(37) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(38) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(39) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi + +(40) CometProject Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(47) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#32, d_year#33, d_moy#34] +(41) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 10] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] - -(49) Filter [codegen id : 10] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] -Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) +(42) CometFilter +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) -(50) Project [codegen id : 10] -Output [1]: [d_date_sk#32] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +(43) CometProject +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Arguments: [d_date_sk#27], [d_date_sk#27] -(51) BroadcastExchange -Input [1]: [d_date_sk#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(44) CometBroadcastExchange +Input [1]: [d_date_sk#27] +Arguments: [d_date_sk#27] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None +(45) CometBroadcastHashJoin +Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [d_date_sk#27] +Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#35] -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#32] +(46) CometProject +Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] +Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] -(54) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#40)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 16] -Input [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(48) ReusedExchange [Reuses operator id: 20] +Output [1]: [item_sk#15] -(56) ReusedExchange [Reuses operator id: 24] -Output [1]: [item_sk#17] +(49) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [item_sk#15] +Arguments: [ws_item_sk#31], [item_sk#15], LeftSemi, BuildRight -(57) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [ws_item_sk#36] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None +(50) CometProject +Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -(58) Project [codegen id : 16] -Output [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Input [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(51) CometExchange +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(59) Exchange -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Arguments: hashpartitioning(ws_bill_customer_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(52) CometSort +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] -(60) Sort [codegen id : 17] -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Arguments: [ws_bill_customer_sk#37 ASC NULLS FIRST], false, 0 +(53) ReusedExchange [Reuses operator id: 34] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] -(61) ReusedExchange [Reuses operator id: 40] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] +(54) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -(62) HashAggregate [codegen id : 20] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] +(55) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) -(63) Filter [codegen id : 20] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) +(56) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] -(64) Project [codegen id : 20] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] +(57) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] -(65) Sort [codegen id : 20] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 +(58) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [c_customer_sk#21] +Arguments: [ws_bill_customer_sk#32], [c_customer_sk#21], LeftSemi -(66) SortMergeJoin [codegen id : 22] -Left keys [1]: [ws_bill_customer_sk#37] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None +(59) CometProject +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -(67) Project [codegen id : 22] -Output [3]: [ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(60) ReusedExchange [Reuses operator id: 44] +Output [1]: [d_date_sk#36] -(68) ReusedExchange [Reuses operator id: 51] -Output [1]: [d_date_sk#41] +(61) CometBroadcastHashJoin +Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [d_date_sk#36] +Arguments: [ws_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(69) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#40] -Right keys [1]: [d_date_sk#41] -Join type: Inner -Join condition: None +(62) CometProject +Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#36] +Arguments: [sales#37], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#37] -(70) Project [codegen id : 22] -Output [1]: [(cast(ws_quantity#38 as decimal(10,0)) * ws_list_price#39) AS sales#42] -Input [4]: [ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40, d_date_sk#41] +(63) CometUnion +Child 0 Input [1]: [sales#30] +Child 1 Input [1]: [sales#37] -(71) Union - -(72) HashAggregate [codegen id : 23] -Input [1]: [sales#35] +(64) CometHashAggregate +Input [1]: [sales#30] Keys: [] -Functions [1]: [partial_sum(sales#35)] -Aggregate Attributes [2]: [sum#43, isEmpty#44] -Results [2]: [sum#45, isEmpty#46] +Functions [1]: [partial_sum(sales#30)] -(73) Exchange -Input [2]: [sum#45, isEmpty#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +(65) CometExchange +Input [2]: [sum#38, isEmpty#39] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(74) HashAggregate [codegen id : 24] -Input [2]: [sum#45, isEmpty#46] +(66) CometHashAggregate +Input [2]: [sum#38, isEmpty#39] Keys: [] -Functions [1]: [sum(sales#35)] -Aggregate Attributes [1]: [sum(sales#35)#47] -Results [1]: [sum(sales#35)#47 AS sum(sales)#48] +Functions [1]: [sum(sales#30)] + +(67) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(sales)#40] ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (93) -+- Exchange (92) - +- * HashAggregate (91) - +- * HashAggregate (90) - +- Exchange (89) - +- * HashAggregate (88) - +- * Project (87) - +- * BroadcastHashJoin Inner BuildRight (86) - :- * Project (80) - : +- * BroadcastHashJoin Inner BuildRight (79) - : :- * Filter (77) - : : +- * ColumnarToRow (76) - : : +- Scan parquet spark_catalog.default.store_sales (75) - : +- ReusedExchange (78) - +- BroadcastExchange (85) - +- * Project (84) - +- * Filter (83) - +- * ColumnarToRow (82) - +- Scan parquet spark_catalog.default.date_dim (81) - - -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] +Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (85) ++- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometHashAggregate (81) + +- CometExchange (80) + +- CometHashAggregate (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometFilter (69) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometNativeScan parquet spark_catalog.default.date_dim (73) + + +(68) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#52)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#44)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 3] -Input [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] +(69) CometFilter +Input [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] +Condition : isnotnull(ss_customer_sk#41) -(77) Filter [codegen id : 3] -Input [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] -Condition : isnotnull(ss_customer_sk#49) +(70) ReusedExchange [Reuses operator id: 30] +Output [1]: [c_customer_sk#45] -(78) ReusedExchange [Reuses operator id: 36] -Output [1]: [c_customer_sk#53] +(71) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] +Right output [1]: [c_customer_sk#45] +Arguments: [ss_customer_sk#41], [c_customer_sk#45], Inner, BuildRight -(79) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#49] -Right keys [1]: [c_customer_sk#53] -Join type: Inner -Join condition: None +(72) CometProject +Input [5]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] +Arguments: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45], [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] -(80) Project [codegen id : 3] -Output [4]: [ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53] -Input [5]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#54, d_year#55] +(73) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#46, d_year#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(82) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#54, d_year#55] - -(83) Filter [codegen id : 2] -Input [2]: [d_date_sk#54, d_year#55] -Condition : (d_year#55 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#54)) - -(84) Project [codegen id : 2] -Output [1]: [d_date_sk#54] -Input [2]: [d_date_sk#54, d_year#55] - -(85) BroadcastExchange -Input [1]: [d_date_sk#54] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -(86) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#52] -Right keys [1]: [d_date_sk#54] -Join type: Inner -Join condition: None - -(87) Project [codegen id : 3] -Output [3]: [ss_quantity#50, ss_sales_price#51, c_customer_sk#53] -Input [5]: [ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53, d_date_sk#54] - -(88) HashAggregate [codegen id : 3] -Input [3]: [ss_quantity#50, ss_sales_price#51, c_customer_sk#53] -Keys [1]: [c_customer_sk#53] -Functions [1]: [partial_sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))] -Aggregate Attributes [2]: [sum#56, isEmpty#57] -Results [3]: [c_customer_sk#53, sum#58, isEmpty#59] - -(89) Exchange -Input [3]: [c_customer_sk#53, sum#58, isEmpty#59] -Arguments: hashpartitioning(c_customer_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(90) HashAggregate [codegen id : 4] -Input [3]: [c_customer_sk#53, sum#58, isEmpty#59] -Keys [1]: [c_customer_sk#53] -Functions [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))#60] -Results [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))#60 AS csales#61] - -(91) HashAggregate [codegen id : 4] -Input [1]: [csales#61] +(74) CometFilter +Input [2]: [d_date_sk#46, d_year#47] +Condition : (d_year#47 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#46)) + +(75) CometProject +Input [2]: [d_date_sk#46, d_year#47] +Arguments: [d_date_sk#46], [d_date_sk#46] + +(76) CometBroadcastExchange +Input [1]: [d_date_sk#46] +Arguments: [d_date_sk#46] + +(77) CometBroadcastHashJoin +Left output [4]: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] +Right output [1]: [d_date_sk#46] +Arguments: [ss_sold_date_sk#44], [d_date_sk#46], Inner, BuildRight + +(78) CometProject +Input [5]: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45, d_date_sk#46] +Arguments: [ss_quantity#42, ss_sales_price#43, c_customer_sk#45], [ss_quantity#42, ss_sales_price#43, c_customer_sk#45] + +(79) CometHashAggregate +Input [3]: [ss_quantity#42, ss_sales_price#43, c_customer_sk#45] +Keys [1]: [c_customer_sk#45] +Functions [1]: [partial_sum((cast(ss_quantity#42 as decimal(10,0)) * ss_sales_price#43))] + +(80) CometExchange +Input [3]: [c_customer_sk#45, sum#48, isEmpty#49] +Arguments: hashpartitioning(c_customer_sk#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(81) CometHashAggregate +Input [3]: [c_customer_sk#45, sum#48, isEmpty#49] +Keys [1]: [c_customer_sk#45] +Functions [1]: [sum((cast(ss_quantity#42 as decimal(10,0)) * ss_sales_price#43))] + +(82) CometHashAggregate +Input [1]: [csales#50] Keys: [] -Functions [1]: [partial_max(csales#61)] -Aggregate Attributes [1]: [max#62] -Results [1]: [max#63] +Functions [1]: [partial_max(csales#50)] -(92) Exchange -Input [1]: [max#63] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +(83) CometExchange +Input [1]: [max#51] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(93) HashAggregate [codegen id : 5] -Input [1]: [max#63] +(84) CometHashAggregate +Input [1]: [max#51] Keys: [] -Functions [1]: [max(csales#61)] -Aggregate Attributes [1]: [max(csales#61)#64] -Results [1]: [max(csales#61)#64 AS tpcds_cmax#65] +Functions [1]: [max(csales#50)] + +(85) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#52] -Subquery:2 Hosting operator id = 63 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/extended.txt new file mode 100644 index 0000000000..a914b98c8e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/extended.txt @@ -0,0 +1,121 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometSort + : +- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 115 out of 117 eligible operators (98%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/simplified.txt index a7ccacb8b9..3604457d42 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/simplified.txt @@ -1,148 +1,91 @@ -WholeStageCodegen (24) - HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (23) - HashAggregate [sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (11) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #2 - WholeStageCodegen (5) - Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - BroadcastHashJoin [cs_item_sk,item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [item_sk] - Filter [cnt] - HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,cnt,count] - InputAdapter - Exchange [_groupingexpression,i_item_sk,d_date] #4 - WholeStageCodegen (3) - HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count] - Project [d_date,i_item_sk,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (9) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (5) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (4) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (3) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [c_customer_sk] #8 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (8) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (10) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow + CometHashAggregate [sum,isEmpty] [sum(sales),sum(sales)] + CometExchange #1 + CometHashAggregate [sales] [sum,isEmpty] + CometUnion [sales] + CometProject [cs_quantity,cs_list_price] [sales] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (22) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (17) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #13 - WholeStageCodegen (16) - Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - BroadcastHashJoin [ws_item_sk,item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (20) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [d_date_sk] #12 + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [ws_quantity,ws_list_price] [sales] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #13 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/explain.txt index fd5c4ba2e7..f4c598eaef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/explain.txt @@ -1,671 +1,600 @@ == Physical Plan == -TakeOrderedAndProject (95) -+- Union (94) - :- * HashAggregate (70) - : +- Exchange (69) - : +- * HashAggregate (68) - : +- * Project (67) - : +- * BroadcastHashJoin Inner BuildRight (66) - : :- * Project (60) - : : +- * BroadcastHashJoin Inner BuildRight (59) - : : :- * SortMergeJoin LeftSemi (46) - : : : :- * Sort (29) - : : : : +- Exchange (28) - : : : : +- * Project (27) - : : : : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (25) - : : : : +- * Project (24) - : : : : +- * Filter (23) - : : : : +- * HashAggregate (22) - : : : : +- Exchange (21) - : : : : +- * HashAggregate (20) - : : : : +- * Project (19) - : : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : : :- * Project (13) - : : : : : +- * BroadcastHashJoin Inner BuildRight (12) - : : : : : :- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : : +- BroadcastExchange (11) - : : : : : +- * Project (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) - : : : : : +- Scan parquet spark_catalog.default.date_dim (7) - : : : : +- BroadcastExchange (17) - : : : : +- * Filter (16) - : : : : +- * ColumnarToRow (15) - : : : : +- Scan parquet spark_catalog.default.item (14) - : : : +- * Sort (45) - : : : +- * Project (44) - : : : +- * Filter (43) - : : : +- * HashAggregate (42) - : : : +- Exchange (41) - : : : +- * HashAggregate (40) - : : : +- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * Project (33) - : : : : +- * Filter (32) - : : : : +- * ColumnarToRow (31) - : : : : +- Scan parquet spark_catalog.default.store_sales (30) - : : : +- BroadcastExchange (37) - : : : +- * Filter (36) - : : : +- * ColumnarToRow (35) - : : : +- Scan parquet spark_catalog.default.customer (34) - : : +- BroadcastExchange (58) - : : +- * SortMergeJoin LeftSemi (57) - : : :- * Sort (51) - : : : +- Exchange (50) - : : : +- * Filter (49) - : : : +- * ColumnarToRow (48) - : : : +- Scan parquet spark_catalog.default.customer (47) - : : +- * Sort (56) - : : +- * Project (55) - : : +- * Filter (54) - : : +- * HashAggregate (53) - : : +- ReusedExchange (52) - : +- BroadcastExchange (65) - : +- * Project (64) - : +- * Filter (63) - : +- * ColumnarToRow (62) - : +- Scan parquet spark_catalog.default.date_dim (61) - +- * HashAggregate (93) - +- Exchange (92) - +- * HashAggregate (91) - +- * Project (90) - +- * BroadcastHashJoin Inner BuildRight (89) - :- * Project (87) - : +- * BroadcastHashJoin Inner BuildRight (86) - : :- * SortMergeJoin LeftSemi (84) - : : :- * Sort (78) - : : : +- Exchange (77) - : : : +- * Project (76) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (75) - : : : :- * Filter (73) - : : : : +- * ColumnarToRow (72) - : : : : +- Scan parquet spark_catalog.default.web_sales (71) - : : : +- ReusedExchange (74) - : : +- * Sort (83) - : : +- * Project (82) - : : +- * Filter (81) - : : +- * HashAggregate (80) - : : +- ReusedExchange (79) - : +- ReusedExchange (85) - +- ReusedExchange (88) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (88) ++- CometTakeOrderedAndProject (87) + +- CometUnion (86) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometSortMergeJoin (40) + : : : :- CometSort (25) + : : : : +- CometExchange (24) + : : : : +- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometHashAggregate (18) + : : : : +- CometExchange (17) + : : : : +- CometHashAggregate (16) + : : : : +- CometProject (15) + : : : : +- CometBroadcastHashJoin (14) + : : : : :- CometProject (10) + : : : : : +- CometBroadcastHashJoin (9) + : : : : : :- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : : +- CometBroadcastExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (5) + : : : : +- CometBroadcastExchange (13) + : : : : +- CometFilter (12) + : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : +- CometSort (39) + : : : +- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometHashAggregate (36) + : : : +- CometExchange (35) + : : : +- CometHashAggregate (34) + : : : +- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) + : : : +- CometBroadcastExchange (31) + : : : +- CometFilter (30) + : : : +- CometNativeScan parquet spark_catalog.default.customer (29) + : : +- CometBroadcastExchange (52) + : : +- CometProject (51) + : : +- CometSortMergeJoin (50) + : : :- CometSort (44) + : : : +- CometExchange (43) + : : : +- CometFilter (42) + : : : +- CometNativeScan parquet spark_catalog.default.customer (41) + : : +- CometSort (49) + : : +- CometProject (48) + : : +- CometFilter (47) + : : +- CometHashAggregate (46) + : : +- ReusedExchange (45) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + +- CometHashAggregate (85) + +- CometExchange (84) + +- CometHashAggregate (83) + +- CometProject (82) + +- CometBroadcastHashJoin (81) + :- CometProject (79) + : +- CometBroadcastHashJoin (78) + : :- CometSortMergeJoin (76) + : : :- CometSort (70) + : : : +- CometExchange (69) + : : : +- CometProject (68) + : : : +- CometBroadcastHashJoin (67) + : : : :- CometFilter (65) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (64) + : : : +- ReusedExchange (66) + : : +- CometSort (75) + : : +- CometProject (74) + : : +- CometFilter (73) + : : +- CometHashAggregate (72) + : : +- ReusedExchange (71) + : +- ReusedExchange (77) + +- ReusedExchange (80) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_bill_customer_sk#1) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] - -(6) Filter [codegen id : 3] +(4) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(7) Scan parquet spark_catalog.default.date_dim +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(9) Filter [codegen id : 1] +(6) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) -(10) Project [codegen id : 1] -Output [2]: [d_date_sk#8, d_date#9] +(7) CometProject Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] -(11) BroadcastExchange +(8) CometBroadcastExchange Input [2]: [d_date_sk#8, d_date#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8, d_date#9] -(12) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(9) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(13) Project [codegen id : 3] -Output [2]: [ss_item_sk#6, d_date#9] +(10) CometProject Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(14) Scan parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] - -(16) Filter [codegen id : 2] +(12) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] Condition : isnotnull(i_item_sk#11) -(17) BroadcastExchange +(13) CometBroadcastExchange Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#11, i_item_desc#12] -(18) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#6] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight -(19) Project [codegen id : 3] -Output [3]: [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] +(15) CometProject Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] -(20) HashAggregate [codegen id : 3] +(16) CometHashAggregate Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#14] -Results [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -(21) Exchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) HashAggregate [codegen id : 4] -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] +(18) CometHashAggregate +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#16] -Results [2]: [i_item_sk#11 AS item_sk#17, count(1)#16 AS cnt#18] -(23) Filter [codegen id : 4] -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +(19) CometFilter +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) -(24) Project [codegen id : 4] -Output [1]: [item_sk#17] -Input [2]: [item_sk#17, cnt#18] +(20) CometProject +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] -(25) BroadcastExchange -Input [1]: [item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [1]: [item_sk#15] +Arguments: [item_sk#15] -(26) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None +(22) CometBroadcastHashJoin +Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight -(27) Project [codegen id : 5] -Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +(23) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(28) Exchange +(24) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(29) Sort [codegen id : 6] +(25) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(30) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(26) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(32) Filter [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +(27) CometFilter +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) -(33) Project [codegen id : 8] -Output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(28) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(34) Scan parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +(29) CometNativeScan parquet spark_catalog.default.customer +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 7] -Input [1]: [c_customer_sk#23] - -(36) Filter [codegen id : 7] -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) - -(37) BroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#19] -Right keys [1]: [c_customer_sk#23] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 8] -Output [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] - -(40) HashAggregate [codegen id : 8] -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [2]: [sum#24, isEmpty#25] -Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(41) Exchange -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 9] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(43) Filter [codegen id : 9] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) - -(44) Project [codegen id : 9] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(45) Sort [codegen id : 9] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(46) SortMergeJoin [codegen id : 17] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(47) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] +(30) CometFilter +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) + +(31) CometBroadcastExchange +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] + +(32) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight + +(33) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] + +(34) CometHashAggregate +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(35) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(36) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(37) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) + +(38) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(39) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(40) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi + +(41) CometNativeScan parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 10] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] - -(49) Filter [codegen id : 10] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Condition : isnotnull(c_customer_sk#32) - -(50) Exchange -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: hashpartitioning(c_customer_sk#32, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) Sort [codegen id : 11] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: [c_customer_sk#32 ASC NULLS FIRST], false, 0 - -(52) ReusedExchange [Reuses operator id: 41] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(53) HashAggregate [codegen id : 14] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(54) Filter [codegen id : 14] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) - -(55) Project [codegen id : 14] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(56) Sort [codegen id : 14] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(57) SortMergeJoin [codegen id : 15] -Left keys [1]: [c_customer_sk#32] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(58) BroadcastExchange -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#32] -Join type: Inner -Join condition: None - -(60) Project [codegen id : 17] -Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34] -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#32, c_first_name#33, c_last_name#34] - -(61) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] +(42) CometFilter +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Condition : isnotnull(c_customer_sk#27) + +(43) CometExchange +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(44) CometSort +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] + +(45) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] + +(46) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(47) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(48) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(49) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(50) CometSortMergeJoin +Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi + +(51) CometProject +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#31] + +(52) CometBroadcastExchange +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] + +(53) CometBroadcastHashJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight + +(54) CometProject +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] + +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(62) ColumnarToRow [codegen id : 16] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] - -(63) Filter [codegen id : 16] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2000)) AND (d_moy#37 = 2)) AND isnotnull(d_date_sk#35)) +(56) CometFilter +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) -(64) Project [codegen id : 16] -Output [1]: [d_date_sk#35] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] +(57) CometProject +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] -(65) BroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +(58) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(66) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#35] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight -(67) Project [codegen id : 17] -Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34, d_date_sk#35] +(60) CometProject +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] -(68) HashAggregate [codegen id : 17] -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] -Keys [2]: [c_last_name#34, c_first_name#33] +(61) CometHashAggregate +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [2]: [sum#38, isEmpty#39] -Results [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -(69) Exchange -Input [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -Arguments: hashpartitioning(c_last_name#34, c_first_name#33, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(70) HashAggregate [codegen id : 18] -Input [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -Keys [2]: [c_last_name#34, c_first_name#33] +(63) CometHashAggregate +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42] -Results [3]: [c_last_name#34, c_first_name#33, sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42 AS sales#43] -(71) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] +(64) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#48)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 23] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] - -(73) Filter [codegen id : 23] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Condition : isnotnull(ws_bill_customer_sk#45) - -(74) ReusedExchange [Reuses operator id: 25] -Output [1]: [item_sk#17] - -(75) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ws_item_sk#44] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None - -(76) Project [codegen id : 23] -Output [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] - -(77) Exchange -Input [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Arguments: hashpartitioning(ws_bill_customer_sk#45, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(78) Sort [codegen id : 24] -Input [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Arguments: [ws_bill_customer_sk#45 ASC NULLS FIRST], false, 0 - -(79) ReusedExchange [Reuses operator id: 41] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(80) HashAggregate [codegen id : 27] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(81) Filter [codegen id : 27] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) - -(82) Project [codegen id : 27] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(83) Sort [codegen id : 27] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(84) SortMergeJoin [codegen id : 35] -Left keys [1]: [ws_bill_customer_sk#45] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(85) ReusedExchange [Reuses operator id: 58] -Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] - -(86) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ws_bill_customer_sk#45] -Right keys [1]: [c_customer_sk#49] -Join type: Inner -Join condition: None - -(87) Project [codegen id : 35] -Output [5]: [ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_first_name#50, c_last_name#51] -Input [7]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_customer_sk#49, c_first_name#50, c_last_name#51] - -(88) ReusedExchange [Reuses operator id: 65] -Output [1]: [d_date_sk#52] - -(89) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ws_sold_date_sk#48] -Right keys [1]: [d_date_sk#52] -Join type: Inner -Join condition: None - -(90) Project [codegen id : 35] -Output [4]: [ws_quantity#46, ws_list_price#47, c_first_name#50, c_last_name#51] -Input [6]: [ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_first_name#50, c_last_name#51, d_date_sk#52] - -(91) HashAggregate [codegen id : 35] -Input [4]: [ws_quantity#46, ws_list_price#47, c_first_name#50, c_last_name#51] -Keys [2]: [c_last_name#51, c_first_name#50] -Functions [1]: [partial_sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))] -Aggregate Attributes [2]: [sum#53, isEmpty#54] -Results [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] - -(92) Exchange -Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(93) HashAggregate [codegen id : 36] -Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Keys [2]: [c_last_name#51, c_first_name#50] -Functions [1]: [sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))] -Aggregate Attributes [1]: [sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))#57] -Results [3]: [c_last_name#51, c_first_name#50, sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))#57 AS sales#58] - -(94) Union - -(95) TakeOrderedAndProject -Input [3]: [c_last_name#34, c_first_name#33, sales#43] -Arguments: 100, [c_last_name#34 ASC NULLS FIRST, c_first_name#33 ASC NULLS FIRST, sales#43 ASC NULLS FIRST], [c_last_name#34, c_first_name#33, sales#43] +(65) CometFilter +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) + +(66) ReusedExchange [Reuses operator id: 21] +Output [1]: [item_sk#15] + +(67) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#15] +Arguments: [ws_item_sk#37], [item_sk#15], LeftSemi, BuildRight + +(68) CometProject +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] + +(69) CometExchange +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(70) CometSort +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] + +(71) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] + +(72) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(73) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(74) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(75) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(76) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#21] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#21], LeftSemi + +(77) ReusedExchange [Reuses operator id: 52] +Output [3]: [c_customer_sk#42, c_first_name#43, c_last_name#44] + +(78) CometBroadcastHashJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#42, c_first_name#43, c_last_name#44] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#42], Inner, BuildRight + +(79) CometProject +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#42, c_first_name#43, c_last_name#44] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44] + +(80) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#45] + +(81) CometBroadcastHashJoin +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44] +Right output [1]: [d_date_sk#45] +Arguments: [ws_sold_date_sk#41], [d_date_sk#45], Inner, BuildRight + +(82) CometProject +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44, d_date_sk#45] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44], [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44] + +(83) CometHashAggregate +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44] +Keys [2]: [c_last_name#44, c_first_name#43] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(84) CometExchange +Input [4]: [c_last_name#44, c_first_name#43, sum#46, isEmpty#47] +Arguments: hashpartitioning(c_last_name#44, c_first_name#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(85) CometHashAggregate +Input [4]: [c_last_name#44, c_first_name#43, sum#46, isEmpty#47] +Keys [2]: [c_last_name#44, c_first_name#43] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(86) CometUnion +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#48] +Child 1 Input [3]: [c_last_name#44, c_first_name#43, sales#49] + +(87) CometTakeOrderedAndProject +Input [3]: [c_last_name#31, c_first_name#30, sales#48] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#48 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#48]), [c_last_name#31, c_first_name#30, sales#48], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#48 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#48] + +(88) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#31, c_first_name#30, sales#48] ===== Subqueries ===== -Subquery:1 Hosting operator id = 43 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (114) -+- Exchange (113) - +- * HashAggregate (112) - +- * HashAggregate (111) - +- Exchange (110) - +- * HashAggregate (109) - +- * Project (108) - +- * BroadcastHashJoin Inner BuildRight (107) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * Filter (98) - : : +- * ColumnarToRow (97) - : : +- Scan parquet spark_catalog.default.store_sales (96) - : +- ReusedExchange (99) - +- BroadcastExchange (106) - +- * Project (105) - +- * Filter (104) - +- * ColumnarToRow (103) - +- Scan parquet spark_catalog.default.date_dim (102) - - -(96) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] +Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometNativeScan parquet spark_catalog.default.date_dim (94) + + +(89) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#62)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#53)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(97) ColumnarToRow [codegen id : 3] -Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] +(90) CometFilter +Input [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] +Condition : isnotnull(ss_customer_sk#50) -(98) Filter [codegen id : 3] -Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] -Condition : isnotnull(ss_customer_sk#59) +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#54] -(99) ReusedExchange [Reuses operator id: 37] -Output [1]: [c_customer_sk#63] +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] +Right output [1]: [c_customer_sk#54] +Arguments: [ss_customer_sk#50], [c_customer_sk#54], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#59] -Right keys [1]: [c_customer_sk#63] -Join type: Inner -Join condition: None +(93) CometProject +Input [5]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] +Arguments: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54], [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] -(101) Project [codegen id : 3] -Output [4]: [ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63] -Input [5]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63] - -(102) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_year#65] +(94) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#55, d_year#56] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#64, d_year#65] - -(104) Filter [codegen id : 2] -Input [2]: [d_date_sk#64, d_year#65] -Condition : (d_year#65 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#64)) - -(105) Project [codegen id : 2] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_year#65] - -(106) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -(107) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#62] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(108) Project [codegen id : 3] -Output [3]: [ss_quantity#60, ss_sales_price#61, c_customer_sk#63] -Input [5]: [ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63, d_date_sk#64] - -(109) HashAggregate [codegen id : 3] -Input [3]: [ss_quantity#60, ss_sales_price#61, c_customer_sk#63] -Keys [1]: [c_customer_sk#63] -Functions [1]: [partial_sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))] -Aggregate Attributes [2]: [sum#66, isEmpty#67] -Results [3]: [c_customer_sk#63, sum#68, isEmpty#69] - -(110) Exchange -Input [3]: [c_customer_sk#63, sum#68, isEmpty#69] -Arguments: hashpartitioning(c_customer_sk#63, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(111) HashAggregate [codegen id : 4] -Input [3]: [c_customer_sk#63, sum#68, isEmpty#69] -Keys [1]: [c_customer_sk#63] -Functions [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))#70] -Results [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))#70 AS csales#71] - -(112) HashAggregate [codegen id : 4] -Input [1]: [csales#71] +(95) CometFilter +Input [2]: [d_date_sk#55, d_year#56] +Condition : (d_year#56 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#55)) + +(96) CometProject +Input [2]: [d_date_sk#55, d_year#56] +Arguments: [d_date_sk#55], [d_date_sk#55] + +(97) CometBroadcastExchange +Input [1]: [d_date_sk#55] +Arguments: [d_date_sk#55] + +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] +Right output [1]: [d_date_sk#55] +Arguments: [ss_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight + +(99) CometProject +Input [5]: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54, d_date_sk#55] +Arguments: [ss_quantity#51, ss_sales_price#52, c_customer_sk#54], [ss_quantity#51, ss_sales_price#52, c_customer_sk#54] + +(100) CometHashAggregate +Input [3]: [ss_quantity#51, ss_sales_price#52, c_customer_sk#54] +Keys [1]: [c_customer_sk#54] +Functions [1]: [partial_sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] + +(101) CometExchange +Input [3]: [c_customer_sk#54, sum#57, isEmpty#58] +Arguments: hashpartitioning(c_customer_sk#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(102) CometHashAggregate +Input [3]: [c_customer_sk#54, sum#57, isEmpty#58] +Keys [1]: [c_customer_sk#54] +Functions [1]: [sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] + +(103) CometHashAggregate +Input [1]: [csales#59] Keys: [] -Functions [1]: [partial_max(csales#71)] -Aggregate Attributes [1]: [max#72] -Results [1]: [max#73] +Functions [1]: [partial_max(csales#59)] -(113) Exchange -Input [1]: [max#73] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +(104) CometExchange +Input [1]: [max#60] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(114) HashAggregate [codegen id : 5] -Input [1]: [max#73] +(105) CometHashAggregate +Input [1]: [max#60] Keys: [] -Functions [1]: [max(csales#71)] -Aggregate Attributes [1]: [max(csales#71)#74] -Results [1]: [max(csales#71)#74 AS tpcds_cmax#75] +Functions [1]: [max(csales#59)] + +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#61] -Subquery:2 Hosting operator id = 54 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/extended.txt new file mode 100644 index 0000000000..bdae9844c6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/extended.txt @@ -0,0 +1,173 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometSort + : : : +- CometProject + : : : +- CometFilter + : : : : +- Subquery + : : : : +- CometColumnarToRow + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometSort + : : +- CometProject + : : +- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometSort + : +- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 165 out of 169 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/simplified.txt index 8099e5fea6..2c389585a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/simplified.txt @@ -1,181 +1,113 @@ -TakeOrderedAndProject [c_last_name,c_first_name,sales] - Union - WholeStageCodegen (18) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #1 - WholeStageCodegen (17) - HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_list_price,c_first_name,c_last_name] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #2 - WholeStageCodegen (5) - Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - BroadcastHashJoin [cs_item_sk,item_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [item_sk] - Filter [cnt] - HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,cnt,count] - InputAdapter - Exchange [_groupingexpression,i_item_sk,d_date] #4 - WholeStageCodegen (3) - HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count] - Project [d_date,i_item_sk,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (9) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (5) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (4) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (3) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [c_customer_sk] #8 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,sales] + CometUnion [c_last_name,c_first_name,sales] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] + CometExchange [c_last_name,c_first_name] #1 + CometHashAggregate [cs_quantity,cs_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (8) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - SortMergeJoin [c_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (11) - Sort [c_customer_sk] - InputAdapter - Exchange [c_customer_sk] #13 - WholeStageCodegen (10) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - WholeStageCodegen (14) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (16) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (36) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #15 - WholeStageCodegen (35) - HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_list_price,c_first_name,c_last_name] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (24) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #16 - WholeStageCodegen (23) - Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - BroadcastHashJoin [ws_item_sk,item_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (27) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] + CometSort [c_customer_sk,c_first_name,c_last_name] + CometExchange [c_customer_sk] #13 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 - InputAdapter - ReusedExchange [d_date_sk] #14 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] + CometExchange [c_last_name,c_first_name] #15 + CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #16 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/explain.txt index 454a150d12..acc9219af4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/explain.txt @@ -1,427 +1,439 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(5) Exchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.store_returns +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(8) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(11) Exchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +(12) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(15) Scan parquet spark_catalog.default.store +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] +(14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) -(17) Filter [codegen id : 5] +(15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(28) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(25) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) -(30) Filter [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] -(31) BroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(33) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(34) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_country)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] -(36) Filter [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] -(37) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(41) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(44) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (73) -+- Exchange (72) - +- * HashAggregate (71) - +- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * Project (64) - : +- * BroadcastHashJoin Inner BuildRight (63) - : :- * Project (61) - : : +- * BroadcastHashJoin Inner BuildRight (60) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * Project (52) - : : : : +- * SortMergeJoin Inner (51) - : : : : :- * Sort (48) - : : : : : +- ReusedExchange (47) - : : : : +- * Sort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.item (56) - : +- ReusedExchange (62) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(48) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(49) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] - -(50) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(51) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(53) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(54) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(56) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometNativeScan parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] + +(57) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) -(58) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) +(59) CometProject +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] -(59) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(60) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(61) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(62) ReusedExchange [Reuses operator id: 31] -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(63) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight -(64) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(65) ReusedExchange [Reuses operator id: 37] -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(66) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(67) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(68) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#40] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(69) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(70) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(71) HashAggregate [codegen id : 10] -Input [1]: [netpaid#31] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#40] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#40] Keys: [] -Functions [1]: [partial_avg(netpaid#31)] -Aggregate Attributes [2]: [sum#42, count#43] -Results [2]: [sum#44, count#45] +Functions [1]: [partial_avg(netpaid#40)] +Aggregate Attributes [2]: [sum#69, count#70] +Results [2]: [sum#71, count#72] + +(75) CometColumnarExchange +Input [2]: [sum#71, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(72) Exchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#71, count#72] -(73) HashAggregate [codegen id : 11] -Input [2]: [sum#44, count#45] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#71, count#72] Keys: [] -Functions [1]: [avg(netpaid#31)] -Aggregate Attributes [1]: [avg(netpaid#31)#46] -Results [1]: [(0.05 * avg(netpaid#31)#46) AS (0.05 * avg(netpaid))#47] +Functions [1]: [avg(netpaid#40)] +Aggregate Attributes [1]: [avg(netpaid#40)#73] +Results [1]: [(0.05 * avg(netpaid#40)#73) AS (0.05 * avg(netpaid))#74] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/extended.txt new file mode 100644 index 0000000000..6b21db37a2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/extended.txt @@ -0,0 +1,96 @@ +Filter +: +- Subquery +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- Project +: +- BroadcastHashJoin +: :- CometColumnarToRow +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometSortMergeJoin +: : : : : :- CometSort +: : : : : : +- CometExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometSort +: : : : : +- CometExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.item +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.customer +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.customer_address ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/simplified.txt index bd820d7de7..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/simplified.txt @@ -1,118 +1,94 @@ -WholeStageCodegen (11) +WholeStageCodegen (4) Filter [paid] Subquery #1 - WholeStageCodegen (11) + WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (9) + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #3 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/explain.txt index 36e95dab7b..dbebe8f96d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/explain.txt @@ -1,427 +1,439 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(5) Exchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.store_returns +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(8) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(11) Exchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +(12) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(15) Scan parquet spark_catalog.default.store +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] +(14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) -(17) Filter [codegen id : 5] +(15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = chiffon ) AND isnotnull(i_item_sk#17)) -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(28) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(25) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) -(30) Filter [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] -(31) BroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(33) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(34) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_country)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] -(36) Filter [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] -(37) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(41) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(44) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (73) -+- Exchange (72) - +- * HashAggregate (71) - +- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * Project (64) - : +- * BroadcastHashJoin Inner BuildRight (63) - : :- * Project (61) - : : +- * BroadcastHashJoin Inner BuildRight (60) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * Project (52) - : : : : +- * SortMergeJoin Inner (51) - : : : : :- * Sort (48) - : : : : : +- ReusedExchange (47) - : : : : +- * Sort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.item (56) - : +- ReusedExchange (62) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(48) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(49) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] - -(50) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(51) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(53) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(54) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(56) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometNativeScan parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] + +(57) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) -(58) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) +(59) CometProject +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] -(59) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(60) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(61) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(62) ReusedExchange [Reuses operator id: 31] -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(63) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight -(64) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(65) ReusedExchange [Reuses operator id: 37] -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(66) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(67) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(68) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#40] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(69) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(70) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(71) HashAggregate [codegen id : 10] -Input [1]: [netpaid#31] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#40] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#40] Keys: [] -Functions [1]: [partial_avg(netpaid#31)] -Aggregate Attributes [2]: [sum#42, count#43] -Results [2]: [sum#44, count#45] +Functions [1]: [partial_avg(netpaid#40)] +Aggregate Attributes [2]: [sum#69, count#70] +Results [2]: [sum#71, count#72] + +(75) CometColumnarExchange +Input [2]: [sum#71, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(72) Exchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#71, count#72] -(73) HashAggregate [codegen id : 11] -Input [2]: [sum#44, count#45] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#71, count#72] Keys: [] -Functions [1]: [avg(netpaid#31)] -Aggregate Attributes [1]: [avg(netpaid#31)#46] -Results [1]: [(0.05 * avg(netpaid#31)#46) AS (0.05 * avg(netpaid))#47] +Functions [1]: [avg(netpaid#40)] +Aggregate Attributes [1]: [avg(netpaid#40)#73] +Results [1]: [(0.05 * avg(netpaid#40)#73) AS (0.05 * avg(netpaid))#74] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/extended.txt new file mode 100644 index 0000000000..6b21db37a2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/extended.txt @@ -0,0 +1,96 @@ +Filter +: +- Subquery +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate +: +- CometColumnarToRow +: +- CometColumnarExchange +: +- HashAggregate +: +- Project +: +- BroadcastHashJoin +: :- CometColumnarToRow +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometSortMergeJoin +: : : : : :- CometSort +: : : : : : +- CometExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometSort +: : : : : +- CometExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_returns +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.item +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.customer +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.customer_address ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 70 out of 86 eligible operators (81%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/simplified.txt index bd820d7de7..af8d5ee7aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/simplified.txt @@ -1,118 +1,94 @@ -WholeStageCodegen (11) +WholeStageCodegen (4) Filter [paid] Subquery #1 - WholeStageCodegen (11) + WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (9) + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #3 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/explain.txt index 07f3e28387..d2e939602b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/explain.txt @@ -1,279 +1,254 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- ReusedExchange (30) - : +- BroadcastExchange (36) - : +- * Filter (35) - : +- * ColumnarToRow (34) - : +- Scan parquet spark_catalog.default.store (33) - +- BroadcastExchange (42) - +- * Filter (41) - +- * ColumnarToRow (40) - +- Scan parquet spark_catalog.default.item (39) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometNativeScan parquet spark_catalog.default.item (34) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +(7) CometProject Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] -(10) Scan parquet spark_catalog.default.catalog_sales +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +(12) CometProject Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] +(15) CometProject Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#16] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +(18) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(23) Scan parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(25) Filter [codegen id : 4] +(20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#19] +(21) CometProject Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] -(27) BroadcastExchange +(22) CometBroadcastExchange Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#19] -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +(24) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] -(30) ReusedExchange [Reuses operator id: 27] +(25) ReusedExchange [Reuses operator id: 22] Output [1]: [d_date_sk#22] -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight -(32) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +(27) CometProject Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -(33) Scan parquet spark_catalog.default.store +(28) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 6] -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] - -(35) Filter [codegen id : 6] +(29) CometFilter Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Condition : isnotnull(s_store_sk#23) -(36) BroadcastExchange +(30) CometProject Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#26, s_store_name#25] + +(31) CometBroadcastExchange +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None +(32) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight -(38) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] +(33) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] -(39) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] +(34) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] +(35) CometFilter +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) -(41) Filter [codegen id : 7] -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Condition : isnotnull(i_item_sk#26) +(36) CometProject +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] -(42) BroadcastExchange -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(37) CometBroadcastExchange +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(38) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight -(44) Project [codegen id : 8] -Output [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] +(39) CometProject +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] -(45) HashAggregate [codegen id : 8] -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(40) CometHashAggregate +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum#29, sum#30, sum#31] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -(46) Exchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometExchange +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(47) HashAggregate [codegen id : 9] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(42) CometHashAggregate +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#35, sum(UnscaledValue(sr_net_loss#10))#36, sum(UnscaledValue(cs_net_profit#14))#37] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#35,17,2) AS store_sales_profit#38, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#36,17,2) AS store_returns_loss#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#37,17,2) AS catalog_sales_profit#40] -(48) TakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] -Arguments: 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] +(43) CometTakeOrderedAndProject +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#26 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_store_id#26,s_store_name#25,store_sales_profit#34,store_returns_loss#35,catalog_sales_profit#36]), [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] + +(44) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/simplified.txt index 7e012d5563..49a567844b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/simplified.txt @@ -1,71 +1,46 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum] - Project [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_net_profit,sr_net_loss,cs_net_profit] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/explain.txt index 718aa301fe..17aa7150fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/explain.txt @@ -1,198 +1,179 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.item (18) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.promotion (24) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.promotion (21) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_bill_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +(8) CometProject Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] +Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] +(14) CometProject Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] +Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -(18) Scan parquet spark_catalog.default.item +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(21) BroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] +(20) CometProject +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] -(24) Scan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(21) CometNativeScan parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) -(26) Filter [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] -(27) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] -(28) BroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_promo_sk#3] -Right keys [1]: [p_promo_sk#17] -Join type: Inner -Join condition: None +(26) CometProject +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] -(30) Project [codegen id : 5] -Output [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] - -(31) HashAggregate [codegen id : 5] -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(27) CometHashAggregate +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -(32) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 6] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [4]: [avg(cs_quantity#4)#36, avg(UnscaledValue(cs_list_price#5))#37, avg(UnscaledValue(cs_coupon_amt#7))#38, avg(UnscaledValue(cs_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(cs_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(cs_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(cs_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(cs_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] -(34) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/extended.txt new file mode 100644 index 0000000000..7382341430 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/simplified.txt index 04314c0c07..55cf7624f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/simplified.txt @@ -1,50 +1,33 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] + CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] + CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/explain.txt index 9f3ff1c51e..75682e065f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/explain.txt @@ -1,198 +1,184 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Expand (30) - +- * Project (29) - +- * BroadcastHashJoin Inner BuildRight (28) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.store (18) - +- BroadcastExchange (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.item (24) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (32) ++- CometTakeOrderedAndProject (31) + +- CometHashAggregate (30) + +- CometExchange (29) + +- CometHashAggregate (28) + +- CometExpand (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.item (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(8) CometProject Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +(14) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(18) Scan parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] +(16) CometFilter Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(20) Filter [codegen id : 3] +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] -(21) BroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(24) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] +(21) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) -(26) Filter [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] -(27) BroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(29) Project [codegen id : 5] -Output [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] -(30) Expand [codegen id : 5] -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] +(27) CometExpand +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] -(31) HashAggregate [codegen id : 5] -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(28) CometHashAggregate +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Results [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -(32) Exchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(29) CometExchange +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 6] -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(30) CometHashAggregate +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#38, avg(UnscaledValue(ss_list_price#5))#39, avg(UnscaledValue(ss_coupon_amt#7))#40, avg(UnscaledValue(ss_sales_price#6))#41] -Results [7]: [i_item_id#19, s_state#20, cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint) AS g_state#42, avg(ss_quantity#4)#38 AS agg1#43, cast((avg(UnscaledValue(ss_list_price#5))#39 / 100.0) as decimal(11,6)) AS agg2#44, cast((avg(UnscaledValue(ss_coupon_amt#7))#40 / 100.0) as decimal(11,6)) AS agg3#45, cast((avg(UnscaledValue(ss_sales_price#6))#41 / 100.0) as decimal(11,6)) AS agg4#46] -(34) TakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] -Arguments: 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] +(31) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] + +(32) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/extended.txt new file mode 100644 index 0000000000..666d08fb88 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/extended.txt @@ -0,0 +1,34 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 31 out of 31 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/simplified.txt index d61e190b9f..ff5133d342 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/simplified.txt @@ -1,50 +1,34 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state,spark_grouping_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Expand [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id,s_state,spark_grouping_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/explain.txt index bfc684b148..985274408b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/explain.txt @@ -1,437 +1,437 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- Exchange (6) -: : : : : +- * HashAggregate (5) -: : : : : +- * Project (4) -: : : : : +- * Filter (3) -: : : : : +- * ColumnarToRow (2) -: : : : : +- Scan parquet spark_catalog.default.store_sales (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- Exchange (16) -: : : : +- * HashAggregate (15) -: : : : +- * Project (14) -: : : : +- * Filter (13) -: : : : +- * ColumnarToRow (12) -: : : : +- Scan parquet spark_catalog.default.store_sales (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- Exchange (28) -: : : +- * HashAggregate (27) -: : : +- * Project (26) -: : : +- * Filter (25) -: : : +- * ColumnarToRow (24) -: : : +- Scan parquet spark_catalog.default.store_sales (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- Exchange (40) -: : +- * HashAggregate (39) -: : +- * Project (38) -: : +- * Filter (37) -: : +- * ColumnarToRow (36) -: : +- Scan parquet spark_catalog.default.store_sales (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- Exchange (52) -: +- * HashAggregate (51) -: +- * Project (50) -: +- * Filter (49) -: +- * ColumnarToRow (48) -: +- Scan parquet spark_catalog.default.store_sales (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * Project (62) - +- * Filter (61) - +- * ColumnarToRow (60) - +- Scan parquet spark_catalog.default.store_sales (59) - - -(1) Scan parquet spark_catalog.default.store_sales +* BroadcastNestedLoopJoin Inner BuildRight (76) +:- * BroadcastNestedLoopJoin Inner BuildRight (63) +: :- * BroadcastNestedLoopJoin Inner BuildRight (50) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (37) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (24) +: : : : :- * CometColumnarToRow (11) +: : : : : +- CometHashAggregate (10) +: : : : : +- CometColumnarExchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * CometColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (23) +: : : : +- * CometColumnarToRow (22) +: : : : +- CometHashAggregate (21) +: : : : +- CometColumnarExchange (20) +: : : : +- * HashAggregate (19) +: : : : +- * HashAggregate (18) +: : : : +- * CometColumnarToRow (17) +: : : : +- CometExchange (16) +: : : : +- CometHashAggregate (15) +: : : : +- CometProject (14) +: : : : +- CometFilter (13) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (12) +: : : +- BroadcastExchange (36) +: : : +- * CometColumnarToRow (35) +: : : +- CometHashAggregate (34) +: : : +- CometColumnarExchange (33) +: : : +- * HashAggregate (32) +: : : +- * HashAggregate (31) +: : : +- * CometColumnarToRow (30) +: : : +- CometExchange (29) +: : : +- CometHashAggregate (28) +: : : +- CometProject (27) +: : : +- CometFilter (26) +: : : +- CometNativeScan parquet spark_catalog.default.store_sales (25) +: : +- BroadcastExchange (49) +: : +- * CometColumnarToRow (48) +: : +- CometHashAggregate (47) +: : +- CometColumnarExchange (46) +: : +- * HashAggregate (45) +: : +- * HashAggregate (44) +: : +- * CometColumnarToRow (43) +: : +- CometExchange (42) +: : +- CometHashAggregate (41) +: : +- CometProject (40) +: : +- CometFilter (39) +: : +- CometNativeScan parquet spark_catalog.default.store_sales (38) +: +- BroadcastExchange (62) +: +- * CometColumnarToRow (61) +: +- CometHashAggregate (60) +: +- CometColumnarExchange (59) +: +- * HashAggregate (58) +: +- * HashAggregate (57) +: +- * CometColumnarToRow (56) +: +- CometExchange (55) +: +- CometHashAggregate (54) +: +- CometProject (53) +: +- CometFilter (52) +: +- CometNativeScan parquet spark_catalog.default.store_sales (51) ++- BroadcastExchange (75) + +- * CometColumnarToRow (74) + +- CometHashAggregate (73) + +- CometColumnarExchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- * CometColumnarToRow (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometProject (66) + +- CometFilter (65) + +- CometNativeScan parquet spark_catalog.default.store_sales (64) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00)))) -(4) Project [codegen id : 1] -Output [1]: [ss_list_price#3] +(3) CometProject Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Arguments: [ss_list_price#3], [ss_list_price#3] -(5) HashAggregate [codegen id : 1] +(4) CometHashAggregate Input [1]: [ss_list_price#3] Keys [1]: [ss_list_price#3] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] -(6) Exchange -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(5) CometExchange +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(7) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] +(6) CometColumnarToRow [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] + +(7) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Keys [1]: [ss_list_price#3] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10] +Results [4]: [ss_list_price#3, sum#6, count#7, count#8] -(8) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] +(8) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [4]: [sum#8, count#9, count#10, count#12] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] +Results [4]: [sum#6, count#7, count#8, count#12] -(9) Exchange -Input [4]: [sum#8, count#9, count#10, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(9) CometColumnarExchange +Input [4]: [sum#6, count#7, count#8, count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(10) HashAggregate [codegen id : 18] -Input [4]: [sum#8, count#9, count#10, count#12] +(10) CometHashAggregate +Input [4]: [sum#6, count#7, count#8, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#7 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(11) Scan parquet spark_catalog.default.store_sales +(11) CometColumnarToRow [codegen id : 12] +Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] + +(12) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] - -(13) Filter [codegen id : 3] +(13) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(14) Project [codegen id : 3] -Output [1]: [ss_list_price#18] +(14) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Arguments: [ss_list_price#18], [ss_list_price#18] -(15) HashAggregate [codegen id : 3] +(15) CometHashAggregate Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] -(16) Exchange -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(17) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] +(17) CometColumnarToRow [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] + +(18) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] +Results [4]: [ss_list_price#18, sum#21, count#22, count#23] -(18) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] +(19) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [4]: [sum#23, count#24, count#25, count#27] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] +Results [4]: [sum#21, count#22, count#23, count#27] -(19) Exchange -Input [4]: [sum#23, count#24, count#25, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(20) CometColumnarExchange +Input [4]: [sum#21, count#22, count#23, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(20) HashAggregate [codegen id : 5] -Input [4]: [sum#23, count#24, count#25, count#27] +(21) CometHashAggregate +Input [4]: [sum#21, count#22, count#23, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#21 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#22 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(21) BroadcastExchange +(22) CometColumnarToRow [codegen id : 3] +Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] + +(23) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(22) BroadcastNestedLoopJoin [codegen id : 18] +(24) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(23) Scan parquet spark_catalog.default.store_sales +(25) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] - -(25) Filter [codegen id : 6] +(26) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(26) Project [codegen id : 6] -Output [1]: [ss_list_price#33] +(27) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Arguments: [ss_list_price#33], [ss_list_price#33] -(27) HashAggregate [codegen id : 6] +(28) CometHashAggregate Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] -(28) Exchange -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(29) CometExchange +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(30) CometColumnarToRow [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -(29) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] +(31) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] +Results [4]: [ss_list_price#33, sum#36, count#37, count#38] -(30) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] +(32) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [4]: [sum#38, count#39, count#40, count#42] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] +Results [4]: [sum#36, count#37, count#38, count#42] -(31) Exchange -Input [4]: [sum#38, count#39, count#40, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(33) CometColumnarExchange +Input [4]: [sum#36, count#37, count#38, count#42] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(32) HashAggregate [codegen id : 8] -Input [4]: [sum#38, count#39, count#40, count#42] +(34) CometHashAggregate +Input [4]: [sum#36, count#37, count#38, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#36 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#37 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(33) BroadcastExchange +(35) CometColumnarToRow [codegen id : 5] +Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] + +(36) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(34) BroadcastNestedLoopJoin [codegen id : 18] +(37) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(35) Scan parquet spark_catalog.default.store_sales +(38) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] ReadSchema: struct -(36) ColumnarToRow [codegen id : 9] -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] - -(37) Filter [codegen id : 9] +(39) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(38) Project [codegen id : 9] -Output [1]: [ss_list_price#48] +(40) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Arguments: [ss_list_price#48], [ss_list_price#48] -(39) HashAggregate [codegen id : 9] +(41) CometHashAggregate Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] -(40) Exchange -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(42) CometExchange +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(43) CometColumnarToRow [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -(41) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] +(44) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] +Results [4]: [ss_list_price#48, sum#51, count#52, count#53] -(42) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] +(45) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [4]: [sum#53, count#54, count#55, count#57] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] +Results [4]: [sum#51, count#52, count#53, count#57] -(43) Exchange -Input [4]: [sum#53, count#54, count#55, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +(46) CometColumnarExchange +Input [4]: [sum#51, count#52, count#53, count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(44) HashAggregate [codegen id : 11] -Input [4]: [sum#53, count#54, count#55, count#57] +(47) CometHashAggregate +Input [4]: [sum#51, count#52, count#53, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#51 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#52 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(45) BroadcastExchange +(48) CometColumnarToRow [codegen id : 7] +Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] + +(49) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(46) BroadcastNestedLoopJoin [codegen id : 18] +(50) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(47) Scan parquet spark_catalog.default.store_sales +(51) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] ReadSchema: struct -(48) ColumnarToRow [codegen id : 12] -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] - -(49) Filter [codegen id : 12] +(52) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(50) Project [codegen id : 12] -Output [1]: [ss_list_price#63] +(53) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Arguments: [ss_list_price#63], [ss_list_price#63] -(51) HashAggregate [codegen id : 12] +(54) CometHashAggregate Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] -(52) Exchange -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(55) CometExchange +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(56) CometColumnarToRow [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -(53) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] +(57) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] +Results [4]: [ss_list_price#63, sum#66, count#67, count#68] -(54) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] +(58) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [4]: [sum#68, count#69, count#70, count#72] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] +Results [4]: [sum#66, count#67, count#68, count#72] -(55) Exchange -Input [4]: [sum#68, count#69, count#70, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +(59) CometColumnarExchange +Input [4]: [sum#66, count#67, count#68, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] -(56) HashAggregate [codegen id : 14] -Input [4]: [sum#68, count#69, count#70, count#72] +(60) CometHashAggregate +Input [4]: [sum#66, count#67, count#68, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#66 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#67 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(57) BroadcastExchange +(61) CometColumnarToRow [codegen id : 9] +Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] + +(62) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(58) BroadcastNestedLoopJoin [codegen id : 18] +(63) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(59) Scan parquet spark_catalog.default.store_sales +(64) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] ReadSchema: struct -(60) ColumnarToRow [codegen id : 15] -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] - -(61) Filter [codegen id : 15] +(65) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(62) Project [codegen id : 15] -Output [1]: [ss_list_price#78] +(66) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Arguments: [ss_list_price#78], [ss_list_price#78] -(63) HashAggregate [codegen id : 15] +(67) CometHashAggregate Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] -(64) Exchange -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(68) CometExchange +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(65) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] +(69) CometColumnarToRow [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] + +(70) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] +Results [4]: [ss_list_price#78, sum#81, count#82, count#83] -(66) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] +(71) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [4]: [sum#83, count#84, count#85, count#87] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] +Results [4]: [sum#81, count#82, count#83, count#87] -(67) Exchange -Input [4]: [sum#83, count#84, count#85, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +(72) CometColumnarExchange +Input [4]: [sum#81, count#82, count#83, count#87] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] -(68) HashAggregate [codegen id : 17] -Input [4]: [sum#83, count#84, count#85, count#87] +(73) CometHashAggregate +Input [4]: [sum#81, count#82, count#83, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#81 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#82 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(69) BroadcastExchange +(74) CometColumnarToRow [codegen id : 11] +Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] + +(75) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(70) BroadcastNestedLoopJoin [codegen id : 18] +(76) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/extended.txt new file mode 100644 index 0000000000..3d8c2f298e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +BroadcastNestedLoopJoin +:- BroadcastNestedLoopJoin +: :- BroadcastNestedLoopJoin +: : :- BroadcastNestedLoopJoin +: : : :- BroadcastNestedLoopJoin +: : : : :- CometColumnarToRow +: : : : : +- CometHashAggregate +: : : : : +- CometColumnarExchange +: : : : : +- HashAggregate +: : : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : : +- CometColumnarToRow +: : : : : +- CometExchange +: : : : : +- CometHashAggregate +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : +- BroadcastExchange +: : : : +- CometColumnarToRow +: : : : +- CometHashAggregate +: : : : +- CometColumnarExchange +: : : : +- HashAggregate +: : : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : : +- CometColumnarToRow +: : : : +- CometExchange +: : : : +- CometHashAggregate +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : +- BroadcastExchange +: : : +- CometColumnarToRow +: : : +- CometHashAggregate +: : : +- CometColumnarExchange +: : : +- HashAggregate +: : : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : : +- CometColumnarToRow +: : : +- CometExchange +: : : +- CometHashAggregate +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : +- BroadcastExchange +: : +- CometColumnarToRow +: : +- CometHashAggregate +: : +- CometColumnarExchange +: : +- HashAggregate +: : +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: : +- CometColumnarToRow +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometHashAggregate +: +- CometColumnarExchange +: +- HashAggregate +: +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] +: +- CometColumnarToRow +: +- CometExchange +: +- CometHashAggregate +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.store_sales ++- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_sales + +Comet accelerated 42 out of 64 eligible operators (65%). Final plan contains 12 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/simplified.txt index 25f9d020f5..a8540a4abb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/simplified.txt @@ -1,111 +1,105 @@ -WholeStageCodegen (18) +WholeStageCodegen (12) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #2 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B1_LP,B1_CNT,B1_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange #4 - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #5 - WholeStageCodegen (3) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B2_LP,B2_CNT,B2_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #4 + WholeStageCodegen (2) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #5 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #6 - WholeStageCodegen (8) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange #7 - WholeStageCodegen (7) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #8 - WholeStageCodegen (6) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B3_LP,B3_CNT,B3_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #7 + WholeStageCodegen (4) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #8 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #9 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] + WholeStageCodegen (7) + CometColumnarToRow InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #11 - WholeStageCodegen (9) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B4_LP,B4_CNT,B4_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #10 + WholeStageCodegen (6) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #11 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #12 - WholeStageCodegen (14) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] + WholeStageCodegen (9) + CometColumnarToRow InputAdapter - Exchange #13 - WholeStageCodegen (13) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #14 - WholeStageCodegen (12) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B5_LP,B5_CNT,B5_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #13 + WholeStageCodegen (8) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #14 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #15 - WholeStageCodegen (17) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - Exchange #16 - WholeStageCodegen (16) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #17 - WholeStageCodegen (15) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B6_LP,B6_CNT,B6_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #16 + WholeStageCodegen (10) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #17 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/explain.txt index a94e9b7806..4b72e29f59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/explain.txt @@ -1,302 +1,273 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin Inner BuildRight (47) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- BroadcastExchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.date_dim (30) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * ColumnarToRow (38) - : +- Scan parquet spark_catalog.default.store (37) - +- BroadcastExchange (46) - +- * Filter (45) - +- * ColumnarToRow (44) - +- Scan parquet spark_catalog.default.item (43) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (19) + : : +- CometBroadcastExchange (28) + : : +- CometProject (27) + : : +- CometFilter (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometNativeScan parquet spark_catalog.default.store (31) + +- CometBroadcastExchange (40) + +- CometProject (39) + +- CometFilter (38) + +- CometNativeScan parquet spark_catalog.default.item (37) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +(7) CometProject Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(10) Scan parquet spark_catalog.default.catalog_sales +(8) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(12) CometProject Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] +(15) CometProject Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#16] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(18) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(23) Scan parquet spark_catalog.default.date_dim +(19) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(25) Filter [codegen id : 4] +(20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#19] +(21) CometProject Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] -(27) BroadcastExchange +(22) CometBroadcastExchange Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#19] -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +(24) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -(30) Scan parquet spark_catalog.default.date_dim +(25) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(32) Filter [codegen id : 5] +(26) CometFilter Input [2]: [d_date_sk#22, d_year#23] Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#22] +(27) CometProject Input [2]: [d_date_sk#22, d_year#23] +Arguments: [d_date_sk#22], [d_date_sk#22] -(34) BroadcastExchange +(28) CometBroadcastExchange Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [d_date_sk#22] -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight -(36) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +(30) CometProject Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(37) Scan parquet spark_catalog.default.store +(31) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 6] -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] - -(39) Filter [codegen id : 6] +(32) CometFilter Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Condition : isnotnull(s_store_sk#24) -(40) BroadcastExchange +(33) CometProject Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#27, s_store_name#26] -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#24] -Join type: Inner -Join condition: None +(34) CometBroadcastExchange +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] -(42) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] +(35) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight -(43) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +(36) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] + +(37) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +(38) CometFilter +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) -(45) Filter [codegen id : 7] -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Condition : isnotnull(i_item_sk#27) +(39) CometProject +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#31, i_item_desc#30] -(46) BroadcastExchange -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(40) CometBroadcastExchange +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] -(47) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#27] -Join type: Inner -Join condition: None +(41) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight -(48) Project [codegen id : 8] -Output [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] +(42) CometProject +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] -(49) HashAggregate [codegen id : 8] -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(43) CometHashAggregate +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum#30, sum#31, sum#32] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -(50) Exchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(44) CometExchange +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(51) HashAggregate [codegen id : 9] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(45) CometHashAggregate +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum(ss_quantity#5)#36, sum(sr_return_quantity#10)#37, sum(cs_quantity#14)#38] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum(ss_quantity#5)#36 AS store_sales_quantity#39, sum(sr_return_quantity#10)#37 AS store_returns_quantity#40, sum(cs_quantity#14)#38 AS catalog_sales_quantity#41] -(52) TakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] -Arguments: 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] +(46) CometTakeOrderedAndProject +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] + +(47) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/extended.txt new file mode 100644 index 0000000000..471731af23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/extended.txt @@ -0,0 +1,49 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 46 out of 46 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/simplified.txt index 01f107ed0a..53a498d595 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/simplified.txt @@ -1,77 +1,49 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum] - Project [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/explain.txt index e05061d827..fd084402f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/explain.txt @@ -1,125 +1,111 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.item (9) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(19) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(17) CometHashAggregate +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] -(21) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, sum_agg#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, sum_agg#16] +(18) CometTakeOrderedAndProject +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,sum_agg#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,sum_agg#15]), [d_year#2, brand_id#13, brand#14, sum_agg#15], 100, 0, [d_year#2 ASC NULLS FIRST, sum_agg#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, sum_agg#15] + +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/simplified.txt index 686e941c28..c19e6caf52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [d_year,brand_id,brand,sum_agg] + CometHashAggregate [sum] [d_year,brand_id,brand,sum_agg,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/explain.txt index f272556a95..f89246ec35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/explain.txt @@ -1,312 +1,283 @@ == Physical Plan == -TakeOrderedAndProject (53) -+- * Project (52) - +- * BroadcastHashJoin Inner BuildRight (51) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_returns (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_address (11) - : : +- BroadcastExchange (37) - : : +- * Filter (36) - : : +- * HashAggregate (35) - : : +- Exchange (34) - : : +- * HashAggregate (33) - : : +- * HashAggregate (32) - : : +- Exchange (31) - : : +- * HashAggregate (30) - : : +- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.web_returns (21) - : : : +- ReusedExchange (24) - : : +- ReusedExchange (27) - : +- BroadcastExchange (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer (40) - +- BroadcastExchange (50) - +- * Project (49) - +- * Filter (48) - +- * ColumnarToRow (47) - +- Scan parquet spark_catalog.default.customer_address (46) - - -(1) Scan parquet spark_catalog.default.web_returns +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometNativeScan parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) + + +(1) CometNativeScan parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_year#6] Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [wr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +(8) CometProject Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] +Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(13) Filter [codegen id : 2] +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(14) BroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [wr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] +(14) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] -(17) HashAggregate [codegen id : 3] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -(18) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 11] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] -(20) Filter [codegen id : 11] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(21) Scan parquet spark_catalog.default.web_returns +(19) CometNativeScan parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(23) Filter [codegen id : 6] +(20) CometFilter Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Condition : isnotnull(wr_returning_addr_sk#2) -(24) ReusedExchange [Reuses operator id: 8] +(21) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#5] -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(26) Project [codegen id : 6] -Output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +(23) CometProject Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] +Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -(27) ReusedExchange [Reuses operator id: 14] -Output [2]: [ca_address_sk#7, ca_state#8] +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#7, ca_state#9] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(29) Project [codegen id : 6] -Output [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] +(26) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] -(30) HashAggregate [codegen id : 6] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(27) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#15] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -(31) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#14] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(32) HashAggregate [codegen id : 7] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(29) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#14] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [2]: [ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] - -(33) HashAggregate [codegen id : 7] -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [ctr_state#13, sum#19, count#20] - -(34) Exchange -Input [3]: [ctr_state#13, sum#19, count#20] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 8] -Input [3]: [ctr_state#13, sum#19, count#20] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] -Aggregate Attributes [1]: [avg(ctr_total_return#14)#21] -Results [2]: [(avg(ctr_total_return#14)#21 * 1.2) AS (avg(ctr_total_return) * 1.2)#22, ctr_state#13 AS ctr_state#13#23] - -(36) Filter [codegen id : 8] -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#22) - -(37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#13#23] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#22) - -(39) Project [codegen id : 11] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] - -(40) Scan parquet spark_catalog.default.customer -Output [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] + +(30) CometHashAggregate +Input [2]: [ctr_state#12, ctr_total_return#13] +Keys [1]: [ctr_state#12] +Functions [1]: [partial_avg(ctr_total_return#13)] + +(31) CometExchange +Input [3]: [ctr_state#12, sum#15, count#16] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(32) CometHashAggregate +Input [3]: [ctr_state#12, sum#15, count#16] +Keys [1]: [ctr_state#12] +Functions [1]: [avg(ctr_total_return#13)] + +(33) CometFilter +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#17) + +(34) CometBroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] + +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_state#12], [ctr_state#12#18], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#17), BuildRight + +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] + +(37) CometNativeScan parquet spark_catalog.default.customer +Output [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] +(38) CometFilter +Input [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#21)) -(42) Filter [codegen id : 9] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#26)) +(39) CometProject +Input [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] +Arguments: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32], [c_customer_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#20, 16, true, false, true) AS c_customer_id#33, c_current_addr_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#22, 10, true, false, true) AS c_salutation#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#25, 1, true, false, true) AS c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#39, c_last_review_date#32] -(43) BroadcastExchange -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(40) CometBroadcastExchange +Input [14]: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] -(44) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [14]: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [ctr_customer_sk#11], [c_customer_sk#19], Inner, BuildRight -(45) Project [codegen id : 11] -Output [14]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] +(42) CometProject +Input [16]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32], [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] -(46) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#38, ca_state#39] +(43) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#40, ca_state#41] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 10] -Input [2]: [ca_address_sk#38, ca_state#39] +(44) CometFilter +Input [2]: [ca_address_sk#40, ca_state#41] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#41, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#40)) -(48) Filter [codegen id : 10] -Input [2]: [ca_address_sk#38, ca_state#39] -Condition : ((isnotnull(ca_state#39) AND (ca_state#39 = GA)) AND isnotnull(ca_address_sk#38)) +(45) CometProject +Input [2]: [ca_address_sk#40, ca_state#41] +Arguments: [ca_address_sk#40], [ca_address_sk#40] -(49) Project [codegen id : 10] -Output [1]: [ca_address_sk#38] -Input [2]: [ca_address_sk#38, ca_state#39] +(46) CometBroadcastExchange +Input [1]: [ca_address_sk#40] +Arguments: [ca_address_sk#40] -(50) BroadcastExchange -Input [1]: [ca_address_sk#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(47) CometBroadcastHashJoin +Left output [14]: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Right output [1]: [ca_address_sk#40] +Arguments: [c_current_addr_sk#21], [ca_address_sk#40], Inner, BuildRight -(51) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#26] -Right keys [1]: [ca_address_sk#38] -Join type: Inner -Join condition: None +(48) CometProject +Input [15]: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ca_address_sk#40] +Arguments: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13], [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] -(52) Project [codegen id : 11] -Output [13]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] -Input [15]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ca_address_sk#38] +(49) CometTakeOrderedAndProject +Input [13]: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#33 ASC NULLS FIRST,c_salutation#34 ASC NULLS FIRST,c_first_name#35 ASC NULLS FIRST,c_last_name#36 ASC NULLS FIRST,c_preferred_cust_flag#37 ASC NULLS FIRST,c_birth_day#26 ASC NULLS FIRST,c_birth_month#27 ASC NULLS FIRST,c_birth_year#28 ASC NULLS FIRST,c_birth_country#29 ASC NULLS FIRST,c_login#38 ASC NULLS FIRST,c_email_address#39 ASC NULLS FIRST,c_last_review_date#32 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#33,c_salutation#34,c_first_name#35,c_last_name#36,c_preferred_cust_flag#37,c_birth_day#26,c_birth_month#27,c_birth_year#28,c_birth_country#29,c_login#38,c_email_address#39,c_last_review_date#32,ctr_total_return#13]), [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13], 100, 0, [c_customer_id#33 ASC NULLS FIRST, c_salutation#34 ASC NULLS FIRST, c_first_name#35 ASC NULLS FIRST, c_last_name#36 ASC NULLS FIRST, c_preferred_cust_flag#37 ASC NULLS FIRST, c_birth_day#26 ASC NULLS FIRST, c_birth_month#27 ASC NULLS FIRST, c_birth_year#28 ASC NULLS FIRST, c_birth_country#29 ASC NULLS FIRST, c_login#38 ASC NULLS FIRST, c_email_address#39 ASC NULLS FIRST, c_last_review_date#32 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] -(53) TakeOrderedAndProject -Input [13]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST, c_salutation#27 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, c_last_name#29 ASC NULLS FIRST, c_preferred_cust_flag#30 ASC NULLS FIRST, c_birth_day#31 ASC NULLS FIRST, c_birth_month#32 ASC NULLS FIRST, c_birth_year#33 ASC NULLS FIRST, c_birth_country#34 ASC NULLS FIRST, c_login#35 ASC NULLS FIRST, c_email_address#36 ASC NULLS FIRST, c_last_review_date#37 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] +(50) CometColumnarToRow [codegen id : 1] +Input [13]: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/extended.txt new file mode 100644 index 0000000000..d2bf365243 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/extended.txt @@ -0,0 +1,58 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 55 out of 55 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/simplified.txt index 8cd53281f4..3b75d209fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/simplified.txt @@ -1,78 +1,52 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - WholeStageCodegen (11) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (3) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - Project [wr_returning_customer_sk,wr_return_amt,ca_state] - BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] - Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk,wr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (8) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (7) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (6) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - Project [wr_returning_customer_sk,wr_return_amt,ca_state] - BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] - Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] + CometBroadcastHashJoin [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ca_address_sk] + CometProject [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometBroadcastHashJoin [ctr_customer_sk,ctr_total_return,c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometProject [ctr_customer_sk,ctr_total_return] + CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] + CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum(UnscaledValue(wr_return_amt))] + CometExchange [wr_returning_customer_sk,ca_state] #1 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 + CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return),ctr_state] + CometExchange [ctr_state] #5 + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [wr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(wr_return_amt))] + CometExchange [wr_returning_customer_sk,ca_state] #6 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #7 + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometBroadcastExchange [ca_address_sk] #8 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/explain.txt index 12ca5d428b..708c0ce9e9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/explain.txt @@ -1,586 +1,509 @@ == Physical Plan == -* Sort (99) -+- Exchange (98) - +- * Project (97) - +- * BroadcastHashJoin Inner BuildRight (96) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin Inner BuildRight (67) - : : :- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * HashAggregate (18) - : : : : : +- Exchange (17) - : : : : : +- * HashAggregate (16) - : : : : : +- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : : : +- BroadcastExchange (34) - : : : : +- * HashAggregate (33) - : : : : +- Exchange (32) - : : : : +- * HashAggregate (31) - : : : : +- * Project (30) - : : : : +- * BroadcastHashJoin Inner BuildRight (29) - : : : : :- * Project (27) - : : : : : +- * BroadcastHashJoin Inner BuildRight (26) - : : : : : :- * Filter (21) - : : : : : : +- * ColumnarToRow (20) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (19) - : : : : : +- BroadcastExchange (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (28) - : : : +- BroadcastExchange (51) - : : : +- * HashAggregate (50) - : : : +- Exchange (49) - : : : +- * HashAggregate (48) - : : : +- * Project (47) - : : : +- * BroadcastHashJoin Inner BuildRight (46) - : : : :- * Project (44) - : : : : +- * BroadcastHashJoin Inner BuildRight (43) - : : : : :- * Filter (38) - : : : : : +- * ColumnarToRow (37) - : : : : : +- Scan parquet spark_catalog.default.store_sales (36) - : : : : +- BroadcastExchange (42) - : : : : +- * Filter (41) - : : : : +- * ColumnarToRow (40) - : : : : +- Scan parquet spark_catalog.default.date_dim (39) - : : : +- ReusedExchange (45) - : : +- BroadcastExchange (66) - : : +- * HashAggregate (65) - : : +- Exchange (64) - : : +- * HashAggregate (63) - : : +- * Project (62) - : : +- * BroadcastHashJoin Inner BuildRight (61) - : : :- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : :- * Filter (56) - : : : : +- * ColumnarToRow (55) - : : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : : +- ReusedExchange (57) - : : +- ReusedExchange (60) - : +- BroadcastExchange (80) - : +- * HashAggregate (79) - : +- Exchange (78) - : +- * HashAggregate (77) - : +- * Project (76) - : +- * BroadcastHashJoin Inner BuildRight (75) - : :- * Project (73) - : : +- * BroadcastHashJoin Inner BuildRight (72) - : : :- * Filter (70) - : : : +- * ColumnarToRow (69) - : : : +- Scan parquet spark_catalog.default.web_sales (68) - : : +- ReusedExchange (71) - : +- ReusedExchange (74) - +- BroadcastExchange (95) - +- * HashAggregate (94) - +- Exchange (93) - +- * HashAggregate (92) - +- * Project (91) - +- * BroadcastHashJoin Inner BuildRight (90) - :- * Project (88) - : +- * BroadcastHashJoin Inner BuildRight (87) - : :- * Filter (85) - : : +- * ColumnarToRow (84) - : : +- Scan parquet spark_catalog.default.web_sales (83) - : +- ReusedExchange (86) - +- ReusedExchange (89) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (90) ++- CometSort (89) + +- CometExchange (88) + +- CometProject (87) + +- CometBroadcastHashJoin (86) + :- CometProject (73) + : +- CometBroadcastHashJoin (72) + : :- CometBroadcastHashJoin (59) + : : :- CometProject (46) + : : : +- CometBroadcastHashJoin (45) + : : : :- CometBroadcastHashJoin (30) + : : : : :- CometHashAggregate (15) + : : : : : +- CometExchange (14) + : : : : : +- CometHashAggregate (13) + : : : : : +- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + : : : : +- CometBroadcastExchange (29) + : : : : +- CometHashAggregate (28) + : : : : +- CometExchange (27) + : : : : +- CometHashAggregate (26) + : : : : +- CometProject (25) + : : : : +- CometBroadcastHashJoin (24) + : : : : :- CometProject (22) + : : : : : +- CometBroadcastHashJoin (21) + : : : : : :- CometFilter (17) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (16) + : : : : : +- CometBroadcastExchange (20) + : : : : : +- CometFilter (19) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (18) + : : : : +- ReusedExchange (23) + : : : +- CometBroadcastExchange (44) + : : : +- CometHashAggregate (43) + : : : +- CometExchange (42) + : : : +- CometHashAggregate (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (32) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (31) + : : : : +- CometBroadcastExchange (35) + : : : : +- CometFilter (34) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (33) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (58) + : : +- CometHashAggregate (57) + : : +- CometExchange (56) + : : +- CometHashAggregate (55) + : : +- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometFilter (48) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) + : : : +- ReusedExchange (49) + : : +- ReusedExchange (52) + : +- CometBroadcastExchange (71) + : +- CometHashAggregate (70) + : +- CometExchange (69) + : +- CometHashAggregate (68) + : +- CometProject (67) + : +- CometBroadcastHashJoin (66) + : :- CometProject (64) + : : +- CometBroadcastHashJoin (63) + : : :- CometFilter (61) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (60) + : : +- ReusedExchange (62) + : +- ReusedExchange (65) + +- CometBroadcastExchange (85) + +- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometProject (81) + +- CometBroadcastHashJoin (80) + :- CometProject (78) + : +- CometBroadcastHashJoin (77) + : :- CometFilter (75) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (74) + : +- ReusedExchange (76) + +- ReusedExchange (79) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_year#5, d_qoy#6] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +(7) CometProject Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_county#8] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [ca_address_sk#7, ca_county#8] Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [ca_address_sk#7, ca_county#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#7, ca_county#8] -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_addr_sk#1] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +Right output [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight -(15) Project [codegen id : 3] -Output [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] +(12) CometProject Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] +Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] -(16) HashAggregate [codegen id : 3] +(13) CometHashAggregate Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#9] -Results [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -(17) Exchange -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(14) CometExchange +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(18) HashAggregate [codegen id : 24] -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] +(15) CometHashAggregate +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#11] -Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS store_sales#12] -(19) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] +(16) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] - -(21) Filter [codegen id : 6] -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_addr_sk#13) +(17) CometFilter +Input [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Condition : isnotnull(ss_addr_sk#10) -(22) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_year#17, d_qoy#18] +(18) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] - -(24) Filter [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Condition : ((((isnotnull(d_qoy#18) AND isnotnull(d_year#17)) AND (d_qoy#18 = 2)) AND (d_year#17 = 2000)) AND isnotnull(d_date_sk#16)) - -(25) BroadcastExchange -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 6] -Output [4]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18] -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_year#17, d_qoy#18] - -(28) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#19, ca_county#20] - -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_addr_sk#13] -Right keys [1]: [ca_address_sk#19] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 6] -Output [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_address_sk#19, ca_county#20] - -(31) HashAggregate [codegen id : 6] -Input [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] - -(32) Exchange -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Arguments: hashpartitioning(ca_county#20, d_qoy#18, d_year#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 7] -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#14))#11] -Results [2]: [ca_county#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#14))#11,17,2) AS store_sales#23] - -(34) BroadcastExchange -Input [2]: [ca_county#20, store_sales#23] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(35) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#20] -Join type: Inner -Join condition: None - -(36) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(19) CometFilter +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) + +(20) CometBroadcastExchange +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [d_date_sk#13, d_year#14, d_qoy#15] + +(21) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Right output [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(22) CometProject +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15], [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] + +(23) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#16, ca_county#17] + +(24) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] +Right output [2]: [ca_address_sk#16, ca_county#17] +Arguments: [ss_addr_sk#10], [ca_address_sk#16], Inner, BuildRight + +(25) CometProject +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_address_sk#16, ca_county#17] +Arguments: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17], [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] + +(26) CometHashAggregate +Input [4]: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#11))] + +(27) CometExchange +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Arguments: hashpartitioning(ca_county#17, d_qoy#15, d_year#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometHashAggregate +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#11))] + +(29) CometBroadcastExchange +Input [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#17, store_sales#19] + +(30) CometBroadcastHashJoin +Left output [3]: [ca_county#8, d_year#5, store_sales#20] +Right output [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#8], [ca_county#17], Inner, BuildRight + +(31) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 10] -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(32) CometFilter +Input [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_addr_sk#21) -(38) Filter [codegen id : 10] -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_addr_sk#24) - -(39) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] +(33) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#24, d_year#25, d_qoy#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 8] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] - -(41) Filter [codegen id : 8] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(42) BroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(43) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 10] -Output [4]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29] -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28, d_qoy#29] - -(45) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#30, ca_county#31] - -(46) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_addr_sk#24] -Right keys [1]: [ca_address_sk#30] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 10] -Output [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_address_sk#30, ca_county#31] - -(48) HashAggregate [codegen id : 10] -Input [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum#32] -Results [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] - -(49) Exchange -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(50) HashAggregate [codegen id : 11] -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#25))#11] -Results [2]: [ca_county#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#25))#11,17,2) AS store_sales#34] - -(51) BroadcastExchange -Input [2]: [ca_county#31, store_sales#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(52) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#20] -Right keys [1]: [ca_county#31] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 24] -Output [5]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34] -Input [7]: [ca_county#8, d_year#5, store_sales#12, ca_county#20, store_sales#23, ca_county#31, store_sales#34] - -(54) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] +(34) CometFilter +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Condition : ((((isnotnull(d_qoy#26) AND isnotnull(d_year#25)) AND (d_qoy#26 = 3)) AND (d_year#25 = 2000)) AND isnotnull(d_date_sk#24)) + +(35) CometBroadcastExchange +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [d_date_sk#24, d_year#25, d_qoy#26] + +(36) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight + +(37) CometProject +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26], [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] + +(38) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#27, ca_county#28] + +(39) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] +Right output [2]: [ca_address_sk#27, ca_county#28] +Arguments: [ss_addr_sk#21], [ca_address_sk#27], Inner, BuildRight + +(40) CometProject +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_address_sk#27, ca_county#28] +Arguments: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28], [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] + +(41) CometHashAggregate +Input [4]: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] + +(42) CometExchange +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Arguments: hashpartitioning(ca_county#28, d_qoy#26, d_year#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(43) CometHashAggregate +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] + +(44) CometBroadcastExchange +Input [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#28, store_sales#30] + +(45) CometBroadcastHashJoin +Left output [5]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19] +Right output [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#17], [ca_county#28], Inner, BuildRight + +(46) CometProject +Input [7]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19, ca_county#28, store_sales#30] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] + +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#37)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#33)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 14] -Input [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] - -(56) Filter [codegen id : 14] -Input [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] -Condition : isnotnull(ws_bill_addr_sk#35) - -(57) ReusedExchange [Reuses operator id: 7] -Output [3]: [d_date_sk#38, d_year#39, d_qoy#40] - -(58) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 14] -Output [4]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#39, d_qoy#40] -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37, d_date_sk#38, d_year#39, d_qoy#40] - -(60) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#41, ca_county#42] - -(61) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_bill_addr_sk#35] -Right keys [1]: [ca_address_sk#41] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 14] -Output [4]: [ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_county#42] -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_address_sk#41, ca_county#42] - -(63) HashAggregate [codegen id : 14] -Input [4]: [ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_county#42] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#36))] -Aggregate Attributes [1]: [sum#43] -Results [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] - -(64) Exchange -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] -Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(65) HashAggregate [codegen id : 15] -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#36))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#36))#45] -Results [2]: [ca_county#42, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#36))#45,17,2) AS web_sales#46] - -(66) BroadcastExchange -Input [2]: [ca_county#42, web_sales#46] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(67) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#42] -Join type: Inner -Join condition: None - -(68) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] +(48) CometFilter +Input [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : isnotnull(ws_bill_addr_sk#31) + +(49) ReusedExchange [Reuses operator id: 5] +Output [3]: [d_date_sk#34, d_year#35, d_qoy#36] + +(50) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Right output [3]: [d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight + +(51) CometProject +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36], [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] + +(52) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#37, ca_county#38] + +(53) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] +Right output [2]: [ca_address_sk#37, ca_county#38] +Arguments: [ws_bill_addr_sk#31], [ca_address_sk#37], Inner, BuildRight + +(54) CometProject +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_address_sk#37, ca_county#38] +Arguments: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38], [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] + +(55) CometHashAggregate +Input [4]: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] + +(56) CometExchange +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Arguments: hashpartitioning(ca_county#38, d_qoy#36, d_year#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(57) CometHashAggregate +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] + +(58) CometBroadcastExchange +Input [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#38, web_sales#40] + +(59) CometBroadcastHashJoin +Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] +Right output [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#8], [ca_county#38], Inner, BuildRight + +(60) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#49)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(69) ColumnarToRow [codegen id : 18] -Input [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] - -(70) Filter [codegen id : 18] -Input [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] -Condition : isnotnull(ws_bill_addr_sk#47) - -(71) ReusedExchange [Reuses operator id: 25] -Output [3]: [d_date_sk#50, d_year#51, d_qoy#52] - -(72) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_sold_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None - -(73) Project [codegen id : 18] -Output [4]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#51, d_qoy#52] -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49, d_date_sk#50, d_year#51, d_qoy#52] - -(74) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#53, ca_county#54] - -(75) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_bill_addr_sk#47] -Right keys [1]: [ca_address_sk#53] -Join type: Inner -Join condition: None - -(76) Project [codegen id : 18] -Output [4]: [ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_county#54] -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_address_sk#53, ca_county#54] - -(77) HashAggregate [codegen id : 18] -Input [4]: [ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_county#54] -Keys [3]: [ca_county#54, d_qoy#52, d_year#51] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#48))] -Aggregate Attributes [1]: [sum#55] -Results [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] - -(78) Exchange -Input [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] -Arguments: hashpartitioning(ca_county#54, d_qoy#52, d_year#51, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(79) HashAggregate [codegen id : 19] -Input [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] -Keys [3]: [ca_county#54, d_qoy#52, d_year#51] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#48))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#48))#45] -Results [2]: [ca_county#54, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#48))#45,17,2) AS web_sales#57] - -(80) BroadcastExchange -Input [2]: [ca_county#54, web_sales#57] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=13] - -(81) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#42] -Right keys [1]: [ca_county#54] -Join type: Inner -Join condition: (CASE WHEN (web_sales#46 > 0.00) THEN (web_sales#57 / web_sales#46) END > CASE WHEN (store_sales#12 > 0.00) THEN (store_sales#23 / store_sales#12) END) - -(82) Project [codegen id : 24] -Output [8]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, web_sales#57] -Input [9]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, ca_county#54, web_sales#57] - -(83) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] +(61) CometFilter +Input [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_addr_sk#41) + +(62) ReusedExchange [Reuses operator id: 20] +Output [3]: [d_date_sk#44, d_year#45, d_qoy#46] + +(63) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Right output [3]: [d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(64) CometProject +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46], [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] + +(65) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#47, ca_county#48] + +(66) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] +Right output [2]: [ca_address_sk#47, ca_county#48] +Arguments: [ws_bill_addr_sk#41], [ca_address_sk#47], Inner, BuildRight + +(67) CometProject +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_address_sk#47, ca_county#48] +Arguments: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48], [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] + +(68) CometHashAggregate +Input [4]: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#42))] + +(69) CometExchange +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Arguments: hashpartitioning(ca_county#48, d_qoy#46, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(70) CometHashAggregate +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#42))] + +(71) CometBroadcastExchange +Input [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#48, web_sales#50] + +(72) CometBroadcastHashJoin +Left output [7]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40] +Right output [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#38], [ca_county#48], Inner, (CASE WHEN (web_sales#40 > 0.00) THEN (web_sales#50 / web_sales#40) END > CASE WHEN (store_sales#20 > 0.00) THEN (store_sales#19 / store_sales#20) END), BuildRight + +(73) CometProject +Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, ca_county#48, web_sales#50] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] + +(74) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#60)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#53)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 22] -Input [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] - -(85) Filter [codegen id : 22] -Input [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] -Condition : isnotnull(ws_bill_addr_sk#58) - -(86) ReusedExchange [Reuses operator id: 42] -Output [3]: [d_date_sk#61, d_year#62, d_qoy#63] - -(87) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#60] -Right keys [1]: [d_date_sk#61] -Join type: Inner -Join condition: None - -(88) Project [codegen id : 22] -Output [4]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#62, d_qoy#63] -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60, d_date_sk#61, d_year#62, d_qoy#63] - -(89) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#64, ca_county#65] - -(90) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_bill_addr_sk#58] -Right keys [1]: [ca_address_sk#64] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 22] -Output [4]: [ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_county#65] -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_address_sk#64, ca_county#65] - -(92) HashAggregate [codegen id : 22] -Input [4]: [ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_county#65] -Keys [3]: [ca_county#65, d_qoy#63, d_year#62] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#59))] -Aggregate Attributes [1]: [sum#66] -Results [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] - -(93) Exchange -Input [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] -Arguments: hashpartitioning(ca_county#65, d_qoy#63, d_year#62, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(94) HashAggregate [codegen id : 23] -Input [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] -Keys [3]: [ca_county#65, d_qoy#63, d_year#62] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#59))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#59))#45] -Results [2]: [ca_county#65, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#59))#45,17,2) AS web_sales#68] - -(95) BroadcastExchange -Input [2]: [ca_county#65, web_sales#68] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=15] - -(96) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#42] -Right keys [1]: [ca_county#65] -Join type: Inner -Join condition: (CASE WHEN (web_sales#57 > 0.00) THEN (web_sales#68 / web_sales#57) END > CASE WHEN (store_sales#23 > 0.00) THEN (store_sales#34 / store_sales#23) END) - -(97) Project [codegen id : 24] -Output [6]: [ca_county#8, d_year#5, (web_sales#57 / web_sales#46) AS web_q1_q2_increase#69, (store_sales#23 / store_sales#12) AS store_q1_q2_increase#70, (web_sales#68 / web_sales#57) AS web_q2_q3_increase#71, (store_sales#34 / store_sales#23) AS store_q2_q3_increase#72] -Input [10]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, web_sales#57, ca_county#65, web_sales#68] - -(98) Exchange -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(99) Sort [codegen id : 25] -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: [ca_county#8 ASC NULLS FIRST], true, 0 +(75) CometFilter +Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_addr_sk#51) + +(76) ReusedExchange [Reuses operator id: 35] +Output [3]: [d_date_sk#54, d_year#55, d_qoy#56] + +(77) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Right output [3]: [d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_sold_date_sk#53], [d_date_sk#54], Inner, BuildRight + +(78) CometProject +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53, d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56], [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] + +(79) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#57, ca_county#58] + +(80) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] +Right output [2]: [ca_address_sk#57, ca_county#58] +Arguments: [ws_bill_addr_sk#51], [ca_address_sk#57], Inner, BuildRight + +(81) CometProject +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_address_sk#57, ca_county#58] +Arguments: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58], [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] + +(82) CometHashAggregate +Input [4]: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#52))] + +(83) CometExchange +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Arguments: hashpartitioning(ca_county#58, d_qoy#56, d_year#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(84) CometHashAggregate +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#52))] + +(85) CometBroadcastExchange +Input [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#58, web_sales#60] + +(86) CometBroadcastHashJoin +Left output [8]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] +Right output [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#38], [ca_county#58], Inner, (CASE WHEN (web_sales#50 > 0.00) THEN (web_sales#60 / web_sales#50) END > CASE WHEN (store_sales#19 > 0.00) THEN (store_sales#30 / store_sales#19) END), BuildRight + +(87) CometProject +Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50, ca_county#58, web_sales#60] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8, d_year#5, (web_sales#50 / web_sales#40) AS web_q1_q2_increase#61, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#62, (web_sales#60 / web_sales#50) AS web_q2_q3_increase#63, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#64] + +(88) CometExchange +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(89) CometSort +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8 ASC NULLS FIRST] + +(90) CometColumnarToRow [codegen id : 1] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/extended.txt new file mode 100644 index 0000000000..f22c7aa1ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/extended.txt @@ -0,0 +1,108 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- CometBroadcastExchange + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 105 out of 105 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/simplified.txt index 5c01b33c93..ef8b87a3b7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/simplified.txt @@ -1,150 +1,92 @@ -WholeStageCodegen (25) - Sort [ca_county] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [ca_county] #1 - WholeStageCodegen (24) - Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - Project [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - Project [ca_county,d_year,store_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - BroadcastHashJoin [ca_county,ca_county] - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #2 - WholeStageCodegen (3) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_county] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #6 - WholeStageCodegen (6) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (11) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #9 - WholeStageCodegen (10) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #12 - WholeStageCodegen (14) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (19) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #14 - WholeStageCodegen (18) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #7 - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (23) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #16 - WholeStageCodegen (22) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #10 - InputAdapter + CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] + CometExchange [ca_county] #1 + CometProject [web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] + CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales,ca_county,web_sales] + CometProject [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,ca_county,web_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales] + CometProject [ca_county,d_year,store_sales,store_sales,store_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales,ca_county,store_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales] + CometHashAggregate [d_qoy,sum] [ca_county,d_year,store_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #2 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [ca_address_sk,ca_county] #4 + CometFilter [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometBroadcastExchange [ca_county,store_sales] #5 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #6 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,store_sales] #8 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #9 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #11 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #12 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #13 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #14 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #15 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #16 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/explain.txt index 65288fb634..a9449edc95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/explain.txt @@ -1,197 +1,173 @@ == Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- * Project (20) - : +- * BroadcastHashJoin Inner BuildRight (19) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.catalog_sales (11) - : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.date_dim (14) - +- ReusedExchange (28) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (30) ++- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#4] +(5) CometProject Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [i_item_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#4] -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Right output [1]: [i_item_sk#4] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(10) Project [codegen id : 6] -Output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +(8) CometProject Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -(11) Scan parquet spark_catalog.default.catalog_sales +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] - -(13) Filter [codegen id : 3] +(10) CometFilter Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Condition : isnotnull(cs_item_sk#6) -(14) Scan parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(16) Filter [codegen id : 2] +(12) CometFilter Input [2]: [d_date_sk#9, d_date#10] Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) -(17) Project [codegen id : 2] -Output [1]: [d_date_sk#9] +(13) CometProject Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(18) BroadcastExchange +(14) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#9] -(19) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(20) Project [codegen id : 3] -Output [2]: [cs_item_sk#6, cs_ext_discount_amt#7] +(16) CometProject Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] +Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] -(21) HashAggregate [codegen id : 3] +(17) CometHashAggregate Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] Keys [1]: [cs_item_sk#6] Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [cs_item_sk#6, sum#13, count#14] -(22) Exchange -Input [3]: [cs_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(18) CometExchange +Input [3]: [cs_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 4] -Input [3]: [cs_item_sk#6, sum#13, count#14] +(19) CometHashAggregate +Input [3]: [cs_item_sk#6, sum#11, count#12] Keys [1]: [cs_item_sk#6] Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(cs_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -(24) Filter [codegen id : 4] -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#16) +(20) CometFilter +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) -(25) BroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [cs_item_sk#6] -Join type: Inner -Join condition: (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#16) +(22) CometBroadcastHashJoin +Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight -(27) Project [codegen id : 6] -Output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] +(23) CometProject +Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] -(28) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#17] +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#9] -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(30) Project [codegen id : 6] -Output [1]: [cs_ext_discount_amt#2] -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#17] +(26) CometProject +Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#9] +Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] -(31) HashAggregate [codegen id : 6] +(27) CometHashAggregate Input [1]: [cs_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#18] -Results [1]: [sum#19] -(32) Exchange -Input [1]: [sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [1]: [sum#14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(33) HashAggregate [codegen id : 7] -Input [1]: [sum#19] +(29) CometHashAggregate +Input [1]: [sum#14] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#20,17,2) AS excess discount amount#21] + +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [excess discount amount#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/extended.txt new file mode 100644 index 0000000000..24ceb24f5b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/simplified.txt index 5e7d3a0c11..5021265f8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/simplified.txt @@ -1,49 +1,32 @@ -WholeStageCodegen (7) - HashAggregate [sum] [sum(UnscaledValue(cs_ext_discount_amt)),excess discount amount,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [cs_ext_discount_amt] [sum,sum] - Project [cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_discount_amt,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] - Project [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk,cs_ext_discount_amt] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Filter [(1.3 * avg(cs_ext_discount_amt))] - HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count] - InputAdapter - Exchange [cs_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] - Project [cs_item_sk,cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - ReusedExchange [d_date_sk] #5 + CometHashAggregate [sum] [excess discount amount,sum(UnscaledValue(cs_ext_discount_amt))] + CometExchange #1 + CometHashAggregate [cs_ext_discount_amt] [sum] + CometProject [cs_ext_discount_amt] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk,(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 + CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] + CometExchange [cs_item_sk] #4 + CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] + CometProject [cs_item_sk,cs_ext_discount_amt] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/explain.txt index 7b096d51d4..44c9175e10 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/explain.txt @@ -1,391 +1,385 @@ == Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometHashAggregate (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometUnion (63) + :- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometFilter (29) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (28) + : : : +- ReusedExchange (30) + : : +- ReusedExchange (33) + : +- CometBroadcastExchange (43) + : +- CometBroadcastHashJoin (42) + : :- CometFilter (37) + : : +- CometNativeScan parquet spark_catalog.default.item (36) + : +- CometBroadcastExchange (41) + : +- CometProject (40) + : +- CometFilter (39) + : +- CometNativeScan parquet spark_catalog.default.item (38) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometBroadcastHashJoin (58) + :- CometProject (56) + : +- CometBroadcastHashJoin (55) + : :- CometProject (53) + : : +- CometBroadcastHashJoin (52) + : : :- CometFilter (50) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (49) + : : +- ReusedExchange (51) + : +- ReusedExchange (54) + +- ReusedExchange (57) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +(8) CometProject Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] +(11) CometProject Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +(14) CometProject Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(18) Scan parquet spark_catalog.default.item +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_manufact_id#11] - -(20) Filter [codegen id : 4] +(16) CometFilter Input [2]: [i_item_sk#10, i_manufact_id#11] Condition : isnotnull(i_item_sk#10) -(21) Scan parquet spark_catalog.default.item -Output [2]: [i_category#12, i_manufact_id#13] +(17) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_category#12, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics )] ReadSchema: struct -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_category#12, i_manufact_id#13] - -(23) Filter [codegen id : 3] -Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) +(18) CometFilter +Input [2]: [i_category#12, i_manufact_id#11] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) -(24) Project [codegen id : 3] -Output [1]: [i_manufact_id#13] -Input [2]: [i_category#12, i_manufact_id#13] +(19) CometProject +Input [2]: [i_category#12, i_manufact_id#11] +Arguments: [i_manufact_id#11#13], [i_manufact_id#11 AS i_manufact_id#11#13] -(25) BroadcastExchange -Input [1]: [i_manufact_id#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(20) CometBroadcastExchange +Input [1]: [i_manufact_id#11#13] +Arguments: [i_manufact_id#11#13] -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_manufact_id#11] -Right keys [1]: [i_manufact_id#13] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_manufact_id#11] +Right output [1]: [i_manufact_id#11#13] +Arguments: [i_manufact_id#11], [i_manufact_id#11#13], LeftSemi, BuildRight -(27) BroadcastExchange +(22) CometBroadcastExchange Input [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#10, i_manufact_id#11] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_manufact_id#11] +(24) CometProject Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] +Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] -(30) HashAggregate [codegen id : 5] +(25) CometHashAggregate Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] Keys [1]: [i_manufact_id#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_manufact_id#11, sum#15] -(31) Exchange -Input [2]: [i_manufact_id#11, sum#15] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [2]: [i_manufact_id#11, sum#14] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [2]: [i_manufact_id#11, sum#15] +(27) CometHashAggregate +Input [2]: [i_manufact_id#11, sum#14] Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +(28) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_manufact_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_manufact_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Keys [1]: [i_manufact_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_manufact_id#25, sum#27] - -(46) Exchange -Input [2]: [i_manufact_id#25, sum#27] -Arguments: hashpartitioning(i_manufact_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_manufact_id#25, sum#27] -Keys [1]: [i_manufact_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_manufact_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +(29) CometFilter +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) + +(30) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#19] + +(31) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(32) CometProject +Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] +Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] + +(33) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#20] + +(34) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +Right output [1]: [ca_address_sk#20] +Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight + +(35) CometProject +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] +Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] + +(36) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#21, i_manufact_id#22] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(37) CometFilter +Input [2]: [i_item_sk#21, i_manufact_id#22] +Condition : isnotnull(i_item_sk#21) + +(38) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_category#12, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(39) CometFilter +Input [2]: [i_category#12, i_manufact_id#11] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) + +(40) CometProject +Input [2]: [i_category#12, i_manufact_id#11] +Arguments: [i_manufact_id#11], [i_manufact_id#11] + +(41) CometBroadcastExchange +Input [1]: [i_manufact_id#11] +Arguments: [i_manufact_id#11] + +(42) CometBroadcastHashJoin +Left output [2]: [i_item_sk#21, i_manufact_id#22] +Right output [1]: [i_manufact_id#11] +Arguments: [i_manufact_id#22], [i_manufact_id#11], LeftSemi, BuildRight + +(43) CometBroadcastExchange +Input [2]: [i_item_sk#21, i_manufact_id#22] +Arguments: [i_item_sk#21, i_manufact_id#22] + +(44) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] +Right output [2]: [i_item_sk#21, i_manufact_id#22] +Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight + +(45) CometProject +Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_manufact_id#22] +Arguments: [cs_ext_sales_price#17, i_manufact_id#22], [cs_ext_sales_price#17, i_manufact_id#22] + +(46) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_manufact_id#22] +Keys [1]: [i_manufact_id#22] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] + +(47) CometExchange +Input [2]: [i_manufact_id#22, sum#23] +Arguments: hashpartitioning(i_manufact_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(48) CometHashAggregate +Input [2]: [i_manufact_id#22, sum#23] +Keys [1]: [i_manufact_id#22] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] + +(49) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_manufact_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_manufact_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_manufact_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_manufact_id#37] -Keys [1]: [i_manufact_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_manufact_id#37, sum#39] - -(61) Exchange -Input [2]: [i_manufact_id#37, sum#39] -Arguments: hashpartitioning(i_manufact_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_manufact_id#37, sum#39] -Keys [1]: [i_manufact_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_manufact_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_manufact_id#11, total_sales#17] +(50) CometFilter +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : (isnotnull(ws_bill_addr_sk#25) AND isnotnull(ws_item_sk#24)) + +(51) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#28] + +(52) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ws_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight + +(53) CometProject +Input [5]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, d_date_sk#28] +Arguments: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26], [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] + +(54) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#29] + +(55) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] +Right output [1]: [ca_address_sk#29] +Arguments: [ws_bill_addr_sk#25], [ca_address_sk#29], Inner, BuildRight + +(56) CometProject +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ca_address_sk#29] +Arguments: [ws_item_sk#24, ws_ext_sales_price#26], [ws_item_sk#24, ws_ext_sales_price#26] + +(57) ReusedExchange [Reuses operator id: 43] +Output [2]: [i_item_sk#30, i_manufact_id#31] + +(58) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#24, ws_ext_sales_price#26] +Right output [2]: [i_item_sk#30, i_manufact_id#31] +Arguments: [ws_item_sk#24], [i_item_sk#30], Inner, BuildRight + +(59) CometProject +Input [4]: [ws_item_sk#24, ws_ext_sales_price#26, i_item_sk#30, i_manufact_id#31] +Arguments: [ws_ext_sales_price#26, i_manufact_id#31], [ws_ext_sales_price#26, i_manufact_id#31] + +(60) CometHashAggregate +Input [2]: [ws_ext_sales_price#26, i_manufact_id#31] +Keys [1]: [i_manufact_id#31] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] + +(61) CometExchange +Input [2]: [i_manufact_id#31, sum#32] +Arguments: hashpartitioning(i_manufact_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(62) CometHashAggregate +Input [2]: [i_manufact_id#31, sum#32] +Keys [1]: [i_manufact_id#31] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] + +(63) CometUnion +Child 0 Input [2]: [i_manufact_id#11, total_sales#33] +Child 1 Input [2]: [i_manufact_id#22, total_sales#34] +Child 2 Input [2]: [i_manufact_id#31, total_sales#35] + +(64) CometHashAggregate +Input [2]: [i_manufact_id#11, total_sales#33] Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_manufact_id#11, sum#44, isEmpty#45] +Functions [1]: [partial_sum(total_sales#33)] -(65) Exchange -Input [3]: [i_manufact_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(65) CometExchange +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(66) HashAggregate [codegen id : 20] -Input [3]: [i_manufact_id#11, sum#44, isEmpty#45] +(66) CometHashAggregate +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_manufact_id#11, sum(total_sales#17)#46 AS total_sales#47] +Functions [1]: [sum(total_sales#33)] + +(67) CometTakeOrderedAndProject +Input [2]: [i_manufact_id#11, total_sales#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#38]), [i_manufact_id#11, total_sales#38], 100, 0, [total_sales#38 ASC NULLS FIRST], [i_manufact_id#11, total_sales#38] -(67) TakeOrderedAndProject -Input [2]: [i_manufact_id#11, total_sales#47] -Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_manufact_id#11, total_sales#47] +(68) CometColumnarToRow [codegen id : 1] +Input [2]: [i_manufact_id#11, total_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/extended.txt new file mode 100644 index 0000000000..a210ba4b5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/extended.txt @@ -0,0 +1,89 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 86 out of 86 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/simplified.txt index 49d9592d2d..02aa7497f4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/simplified.txt @@ -1,101 +1,70 @@ -TakeOrderedAndProject [total_sales,i_manufact_id] - WholeStageCodegen (20) - HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (19) - HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #2 - WholeStageCodegen (5) - HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_manufact_id,i_manufact_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_manufact_id] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_category,i_manufact_id] - WholeStageCodegen (12) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #7 - WholeStageCodegen (11) - HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_manufact_id] #5 - WholeStageCodegen (18) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #8 - WholeStageCodegen (17) - HashAggregate [i_manufact_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_manufact_id] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_manufact_id,total_sales] + CometHashAggregate [sum,isEmpty] [i_manufact_id,total_sales,sum(total_sales)] + CometExchange [i_manufact_id] #1 + CometHashAggregate [total_sales] [i_manufact_id,sum,isEmpty] + CometUnion [i_manufact_id,total_sales] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_manufact_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_manufact_id,sum] + CometProject [ss_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_manufact_id] #5 + CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] + CometFilter [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [i_manufact_id] #6 + CometProject [i_manufact_id] [i_manufact_id] + CometFilter [i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact_id] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_manufact_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] + CometProject [cs_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + CometBroadcastExchange [i_item_sk,i_manufact_id] #8 + CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] + CometFilter [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [i_manufact_id] #9 + CometProject [i_manufact_id] + CometFilter [i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact_id] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_manufact_id] #10 + CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] + CometProject [ws_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/explain.txt index 03e931787b..3df476759f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/explain.txt @@ -1,208 +1,189 @@ == Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +(8) CometProject Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#9, s_county#10] Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] +(11) CometProject Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#9] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +(14) CometProject Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] +(17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#11] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] +(20) CometProject Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 15) AND (cnt#18 <= 20)) +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(25) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST], true, 0 +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/simplified.txt index 048a2e5a32..66a94b088b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/simplified.txt @@ -1,54 +1,35 @@ -WholeStageCodegen (7) - Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/explain.txt index 6140e1584c..6bf31cf119 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/explain.txt @@ -1,267 +1,269 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] +(6) CometProject Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] +(9) CometProject Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#6] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#13] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None -(29) Filter [codegen id : 9] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(30) Project [codegen id : 9] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] +(30) CometFilter +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : isnotnull(ca_address_sk#15) -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) +(31) CometProject +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15, ca_state#17], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#17] -(34) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#15, ca_state#17] -(35) BroadcastHashJoin [codegen id : 9] +(33) BroadcastExchange +Input [2]: [ca_address_sk#15, ca_state#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(36) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#17] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#17] -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(37) CometFilter +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Condition : isnotnull(cd_demo_sk#18) + +(38) CometProject +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (40) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(41) BroadcastHashJoin [codegen id : 9] +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#18] Join type: Inner Join condition: None -(42) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(43) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#22), partial_max(cd_dep_count#22), partial_avg(cd_dep_count#22), partial_min(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_avg(cd_dep_employed_count#23), partial_min(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_avg(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, min#26, max#27, sum#28, count#29, min#30, max#31, sum#32, count#33, min#34, max#35, sum#36, count#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] - -(44) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), min(cd_dep_count#22), max(cd_dep_count#22), avg(cd_dep_count#22), min(cd_dep_employed_count#23), max(cd_dep_employed_count#23), avg(cd_dep_employed_count#23), min(cd_dep_college_count#24), max(cd_dep_college_count#24), avg(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, min(cd_dep_count#22)#52, max(cd_dep_count#22)#53, avg(cd_dep_count#22)#54, min(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, avg(cd_dep_employed_count#23)#57, min(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, avg(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, count(1)#51 AS cnt1#61, min(cd_dep_count#22)#52 AS min(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, avg(cd_dep_count#22)#54 AS avg(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, min(cd_dep_employed_count#23)#55 AS min(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, avg(cd_dep_employed_count#23)#57 AS avg(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, min(cd_dep_college_count#24)#58 AS min(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, avg(cd_dep_college_count#24)#60 AS avg(cd_dep_college_count)#72, cd_dep_count#22] - -(46) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72, cd_dep_count#22] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72] +(42) Project [codegen id : 5] +Output [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Input [8]: [c_current_cdemo_sk#4, ca_state#17, cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#21), partial_max(cd_dep_count#21), partial_avg(cd_dep_count#21), partial_min(cd_dep_employed_count#22), partial_max(cd_dep_employed_count#22), partial_avg(cd_dep_employed_count#22), partial_min(cd_dep_college_count#23), partial_max(cd_dep_college_count#23), partial_avg(cd_dep_college_count#23)] +Aggregate Attributes [13]: [count#26, min#27, max#28, sum#29, count#30, min#31, max#32, sum#33, count#34, min#35, max#36, sum#37, count#38] +Results [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] + +(44) CometColumnarExchange +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] +Arguments: hashpartitioning(ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [count(1), min(cd_dep_count#21), max(cd_dep_count#21), avg(cd_dep_count#21), min(cd_dep_employed_count#22), max(cd_dep_employed_count#22), avg(cd_dep_employed_count#22), min(cd_dep_college_count#23), max(cd_dep_college_count#23), avg(cd_dep_college_count#23)] +Aggregate Attributes [10]: [count(1)#52, min(cd_dep_count#21)#53, max(cd_dep_count#21)#54, avg(cd_dep_count#21)#55, min(cd_dep_employed_count#22)#56, max(cd_dep_employed_count#22)#57, avg(cd_dep_employed_count#22)#58, min(cd_dep_college_count#23)#59, max(cd_dep_college_count#23)#60, avg(cd_dep_college_count#23)#61] +Results [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, count(1)#52 AS cnt1#62, min(cd_dep_count#21)#53 AS min(cd_dep_count)#63, max(cd_dep_count#21)#54 AS max(cd_dep_count)#64, avg(cd_dep_count#21)#55 AS avg(cd_dep_count)#65, cd_dep_employed_count#22, count(1)#52 AS cnt2#66, min(cd_dep_employed_count#22)#56 AS min(cd_dep_employed_count)#67, max(cd_dep_employed_count#22)#57 AS max(cd_dep_employed_count)#68, avg(cd_dep_employed_count#22)#58 AS avg(cd_dep_employed_count)#69, cd_dep_college_count#23, count(1)#52 AS cnt3#70, min(cd_dep_college_count#23)#59 AS min(cd_dep_college_count)#71, max(cd_dep_college_count#23)#60 AS max(cd_dep_college_count)#72, avg(cd_dep_college_count#23)#61 AS avg(cd_dep_college_count)#73, cd_dep_count#21] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cnt1#62, min(cd_dep_count)#63, max(cd_dep_count)#64, avg(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, min(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, avg(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, min(cd_dep_college_count)#71, max(cd_dep_college_count)#72, avg(cd_dep_college_count)#73, cd_dep_count#21] +Arguments: 100, [ca_state#17 ASC NULLS FIRST, cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_dep_count#21 ASC NULLS FIRST, cd_dep_employed_count#22 ASC NULLS FIRST, cd_dep_college_count#23 ASC NULLS FIRST], [ca_state#17, cd_gender#24, cd_marital_status#25, cnt1#62, min(cd_dep_count)#63, max(cd_dep_count)#64, avg(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, min(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, avg(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, min(cd_dep_college_count)#71, max(cd_dep_college_count)#72, avg(cd_dep_college_count)#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/simplified.txt index c00f3f58f0..d7de10ae30 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/simplified.txt @@ -1,70 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/explain.txt index 806e421651..d31e815876 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/explain.txt @@ -1,184 +1,170 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- Window (30) - +- * Sort (29) - +- Exchange (28) - +- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * Expand (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (21) - +- * Project (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.store (17) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (30) ++- * Project (29) + +- Window (28) + +- * CometColumnarToRow (27) + +- CometSort (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#6, d_year#7] Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +(8) CometProject Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(16) Project [codegen id : 4] -Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) -(19) Filter [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(20) Project [codegen id : 3] -Output [1]: [s_store_sk#11] -Input [2]: [s_store_sk#11, s_state#12] +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] -(21) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] -(23) Project [codegen id : 4] -Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] +(21) CometExpand +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] -(24) Expand [codegen id : 4] -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(25) HashAggregate [codegen id : 4] -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(22) CometHashAggregate +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#16, sum#17] -Results [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -(26) Exchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 5] -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(24) CometHashAggregate +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#20, sum(UnscaledValue(ss_ext_sales_price#3))#21] -Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#13, i_class#14, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint) = 0) THEN i_category#13 END AS _w2#26] -(28) Exchange -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometExchange +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: hashpartitioning(_w1#23, _w2#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(26) CometSort +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24], [_w1#23 ASC NULLS FIRST, _w2#24 ASC NULLS FIRST, _w0#22 ASC NULLS FIRST] -(29) Sort [codegen id : 6] -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST], false, 0 +(27) CometColumnarToRow [codegen id : 1] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] -(30) Window -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [rank(_w0#24) windowspecdefinition(_w1#25, _w2#26, _w0#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#27], [_w1#25, _w2#26], [_w0#24 ASC NULLS FIRST] +(28) Window +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [rank(_w0#22) windowspecdefinition(_w1#23, _w2#24, _w0#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#23, _w2#24], [_w0#22 ASC NULLS FIRST] -(31) Project [codegen id : 7] -Output [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Input [8]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] +(29) Project [codegen id : 2] +Output [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Input [8]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24, rank_within_parent#25] -(32) TakeOrderedAndProject -Input [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] +(30) TakeOrderedAndProject +Input [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Arguments: 100, [lochierarchy#21 DESC NULLS LAST, CASE WHEN (lochierarchy#21 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/extended.txt new file mode 100644 index 0000000000..d49f29e436 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/extended.txt @@ -0,0 +1,32 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 26 out of 29 eligible operators (89%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/simplified.txt index 61d30a3e81..cec70668fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/simplified.txt @@ -1,49 +1,34 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (6) - Sort [_w1,_w2,_w0] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - Expand [ss_ext_sales_price,ss_net_profit,i_category,i_class] - Project [ss_ext_sales_price,ss_net_profit,i_category,i_class] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometExchange [_w1,_w2] #1 + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),spark_grouping_id] + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,spark_grouping_id,sum,sum] + CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] + CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #5 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/explain.txt index 326f4cb338..4a3ef391b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/explain.txt @@ -1,169 +1,150 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildLeft (24) - :- BroadcastExchange (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.inventory (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) - +- Scan parquet spark_catalog.default.catalog_sales (20) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (26) ++- CometTakeOrderedAndProject (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.inventory (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + +- CometProject (19) + +- CometFilter (18) + +- CometNativeScan parquet spark_catalog.default.catalog_sales (17) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,694,808,940]), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +(3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(5) Scan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +(5) CometFilter +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(6) CometProject +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] -(9) BroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(7) CometBroadcastExchange +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [inv_item_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight -(11) Project [codegen id : 3] -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] +(9) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(12) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] +(10) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] +(11) CometFilter +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) -(14) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) +(12) CometProject +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] +(13) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(16) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(14) CometBroadcastHashJoin +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [inv_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(18) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] +(16) CometBroadcastExchange +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(19) BroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] +(17) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(21) ColumnarToRow -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] - -(22) Filter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) +(18) CometFilter +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) -(23) Project -Output [1]: [cs_item_sk#11] -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] +(19) CometProject +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#11] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft -(25) Project [codegen id : 4] -Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] +(21) CometProject +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) HashAggregate [codegen id : 4] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(22) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(27) Exchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(24) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(29) TakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +(25) CometTakeOrderedAndProject +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/extended.txt new file mode 100644 index 0000000000..92c5233469 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.inventory + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_sales + +Comet accelerated 25 out of 25 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/simplified.txt index 2d05feb776..d8d49b93e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/simplified.txt @@ -1,42 +1,28 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - WholeStageCodegen (5) - HashAggregate [i_item_id,i_item_desc,i_current_price] - InputAdapter - Exchange [i_item_id,i_item_desc,i_current_price] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_current_price] - Project [i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [i_item_sk,cs_item_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (3) - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - Project [cs_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometExchange [i_item_id,i_item_desc,i_current_price] #1 + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/explain.txt index d5e6da17aa..6841fdcbef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/explain.txt @@ -1,307 +1,271 @@ == Physical Plan == -* HashAggregate (51) -+- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin LeftSemi BuildRight (47) - :- * BroadcastHashJoin LeftSemi BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - +- BroadcastExchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (48) ++- CometHashAggregate (47) + +- CometExchange (46) + +- CometHashAggregate (45) + +- CometProject (44) + +- CometBroadcastHashJoin (43) + :- CometBroadcastHashJoin (30) + : :- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : +- CometBroadcastExchange (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- CometBroadcastExchange (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (31) + : +- ReusedExchange (33) + +- ReusedExchange (36) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#3, d_date#4] +(5) CometProject Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#3, d_date#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#3, d_date#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#2] -Right keys [1]: [d_date_sk#3] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ss_customer_sk#1, d_date#4] +(8) CometProject Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(11) Scan parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] + +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(17) HashAggregate [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(18) Exchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 12] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(18) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] - -(22) Filter [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) +(19) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#11, d_date#12] +(20) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(25) Project [codegen id : 6] -Output [2]: [cs_bill_customer_sk#9, d_date#12] -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] +(22) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(26) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] +(23) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_bill_customer_sk#9] -Right keys [1]: [c_customer_sk#13] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(28) Project [codegen id : 6] -Output [3]: [c_last_name#15, c_first_name#14, d_date#12] -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] +(25) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] -(29) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(26) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(30) Exchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(27) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) HashAggregate [codegen id : 7] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(28) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(32) BroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] +(29) CometBroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] -(33) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)] -Join type: LeftSemi -Join condition: None +(30) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight -(34) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(31) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(32) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(36) Filter [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] -(37) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#18, d_date#19] +(34) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#17] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None +(35) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] -(39) Project [codegen id : 10] -Output [2]: [ws_bill_customer_sk#16, d_date#19] -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] +(36) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] -(40) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(37) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#16] -Right keys [1]: [c_customer_sk#20] -Join type: Inner -Join condition: None +(38) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] -(42) Project [codegen id : 10] -Output [3]: [c_last_name#22, c_first_name#21, d_date#19] -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] - -(43) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(39) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(44) Exchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(40) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(45) HashAggregate [codegen id : 11] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(46) BroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=7] +(42) CometBroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] -(47) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)] -Join type: LeftSemi -Join condition: None +(43) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight -(48) Project [codegen id : 12] -Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(44) CometProject +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(49) HashAggregate [codegen id : 12] +(45) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] -(50) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [1]: [count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(51) HashAggregate [codegen id : 13] -Input [1]: [count#24] +(47) CometHashAggregate +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS count(1)#26] + +(48) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/extended.txt new file mode 100644 index 0000000000..ab9f89f4f0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/extended.txt @@ -0,0 +1,62 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 59 out of 59 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/simplified.txt index 9122ac943b..b912edf4d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/simplified.txt @@ -1,77 +1,50 @@ -WholeStageCodegen (13) - HashAggregate [count] [count(1),count(1),count] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (12) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometHashAggregate [count] [count(1),count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/explain.txt index e203fa858c..e0b2fab87d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/explain.txt @@ -1,301 +1,266 @@ == Physical Plan == -* Sort (52) -+- Exchange (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (27) - : +- * Filter (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.warehouse (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Filter (30) - : : : +- * ColumnarToRow (29) - : : : +- Scan parquet spark_catalog.default.inventory (28) - : : +- ReusedExchange (31) - : +- ReusedExchange (34) - +- BroadcastExchange (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.date_dim (37) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_sk#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [i_item_sk#5] Condition : isnotnull(i_item_sk#5) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [i_item_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#5] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight -(9) Project [codegen id : 4] -Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +(7) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight -(15) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +(12) CometProject Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) -(19) Project [codegen id : 3] -Output [2]: [d_date_sk#8, d_moy#10] +(15) CometProject Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [2]: [d_date_sk#8, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#8, d_moy#10] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +(18) CometProject Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -(23) HashAggregate [codegen id : 4] +(19) CometHashAggregate Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -(24) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 10] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] -(26) Filter [codegen id : 10] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END -(27) Project [codegen id : 10] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] -(28) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] +(24) CometNativeScan parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#29)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(30) Filter [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) -(31) ReusedExchange [Reuses operator id: 7] -Output [1]: [i_item_sk#30] +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] -(32) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_item_sk#26] -Right keys [1]: [i_item_sk#30] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight -(33) Project [codegen id : 8] -Output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -(34) ReusedExchange [Reuses operator id: 13] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_warehouse_sk#27] -Right keys [1]: [w_warehouse_sk#31] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight -(36) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -(37) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(39) Filter [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(40) Project [codegen id : 7] -Output [2]: [d_date_sk#33, d_moy#35] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(41) BroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_date_sk#29] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] - -(44) HashAggregate [codegen id : 8] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(45) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(46) HashAggregate [codegen id : 9] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#28)#22 AS mean#24] - -(47) Filter [codegen id : 9] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END - -(48) Project [codegen id : 9] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#24 AS mean#46, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#47] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] - -(49) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(51) Exchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) Sort [codegen id : 11] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST], true, 0 +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/extended.txt new file mode 100644 index 0000000000..dab448e5d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/simplified.txt index 269ca30df4..cdbd0d9063 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/simplified.txt @@ -1,77 +1,49 @@ -WholeStageCodegen (11) - Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (10) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (8) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - ReusedExchange [i_item_sk] #3 - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/explain.txt index f7800f1f97..62b5730314 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/explain.txt @@ -1,301 +1,266 @@ == Physical Plan == -* Sort (52) -+- Exchange (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (27) - : +- * Filter (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.warehouse (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Filter (30) - : : : +- * ColumnarToRow (29) - : : : +- Scan parquet spark_catalog.default.inventory (28) - : : +- ReusedExchange (31) - : +- ReusedExchange (34) - +- BroadcastExchange (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.date_dim (37) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_sk#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [i_item_sk#5] Condition : isnotnull(i_item_sk#5) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [i_item_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#5] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight -(9) Project [codegen id : 4] -Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +(7) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight -(15) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +(12) CometProject Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) -(19) Project [codegen id : 3] -Output [2]: [d_date_sk#8, d_moy#10] +(15) CometProject Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [2]: [d_date_sk#8, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#8, d_moy#10] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +(18) CometProject Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -(23) HashAggregate [codegen id : 4] +(19) CometHashAggregate Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -(24) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 10] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] -(26) Filter [codegen id : 10] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : (CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END AND CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.5) END) +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) -(27) Project [codegen id : 10] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] -(28) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] +(24) CometNativeScan parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#29)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(30) Filter [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) -(31) ReusedExchange [Reuses operator id: 7] -Output [1]: [i_item_sk#30] +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] -(32) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_item_sk#26] -Right keys [1]: [i_item_sk#30] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight -(33) Project [codegen id : 8] -Output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -(34) ReusedExchange [Reuses operator id: 13] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_warehouse_sk#27] -Right keys [1]: [w_warehouse_sk#31] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight -(36) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -(37) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(39) Filter [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(40) Project [codegen id : 7] -Output [2]: [d_date_sk#33, d_moy#35] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(41) BroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_date_sk#29] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] - -(44) HashAggregate [codegen id : 8] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(45) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(46) HashAggregate [codegen id : 9] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#28)#22 AS mean#24] - -(47) Filter [codegen id : 9] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END - -(48) Project [codegen id : 9] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#24 AS mean#46, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#47] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] - -(49) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(51) Exchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) Sort [codegen id : 11] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST], true, 0 +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/extended.txt new file mode 100644 index 0000000000..dab448e5d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/simplified.txt index 269ca30df4..cdbd0d9063 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/simplified.txt @@ -1,77 +1,49 @@ -WholeStageCodegen (11) - Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (10) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (8) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - ReusedExchange [i_item_sk] #3 - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/explain.txt index b21f0f54f2..bb574f8279 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/explain.txt @@ -1,616 +1,565 @@ == Physical Plan == -TakeOrderedAndProject (105) -+- * Project (104) - +- * BroadcastHashJoin Inner BuildRight (103) - :- * Project (89) - : +- * BroadcastHashJoin Inner BuildRight (88) - : :- * Project (70) - : : +- * BroadcastHashJoin Inner BuildRight (69) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * BroadcastHashJoin Inner BuildRight (36) - : : : : :- * Filter (19) - : : : : : +- * HashAggregate (18) - : : : : : +- Exchange (17) - : : : : : +- * HashAggregate (16) - : : : : : +- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : : : +- BroadcastExchange (35) - : : : : +- * HashAggregate (34) - : : : : +- Exchange (33) - : : : : +- * HashAggregate (32) - : : : : +- * Project (31) - : : : : +- * BroadcastHashJoin Inner BuildRight (30) - : : : : :- * Project (25) - : : : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : : : :- * Filter (22) - : : : : : : +- * ColumnarToRow (21) - : : : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : : : +- ReusedExchange (23) - : : : : +- BroadcastExchange (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) - : : : : +- Scan parquet spark_catalog.default.date_dim (26) - : : : +- BroadcastExchange (53) - : : : +- * Filter (52) - : : : +- * HashAggregate (51) - : : : +- Exchange (50) - : : : +- * HashAggregate (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (39) - : : : : : +- * ColumnarToRow (38) - : : : : : +- Scan parquet spark_catalog.default.customer (37) - : : : : +- BroadcastExchange (43) - : : : : +- * Filter (42) - : : : : +- * ColumnarToRow (41) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (40) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (68) - : : +- * HashAggregate (67) - : : +- Exchange (66) - : : +- * HashAggregate (65) - : : +- * Project (64) - : : +- * BroadcastHashJoin Inner BuildRight (63) - : : :- * Project (61) - : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : :- * Filter (58) - : : : : +- * ColumnarToRow (57) - : : : : +- Scan parquet spark_catalog.default.customer (56) - : : : +- ReusedExchange (59) - : : +- ReusedExchange (62) - : +- BroadcastExchange (87) - : +- * Filter (86) - : +- * HashAggregate (85) - : +- Exchange (84) - : +- * HashAggregate (83) - : +- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * Project (79) - : : +- * BroadcastHashJoin Inner BuildRight (78) - : : :- * Filter (73) - : : : +- * ColumnarToRow (72) - : : : +- Scan parquet spark_catalog.default.customer (71) - : : +- BroadcastExchange (77) - : : +- * Filter (76) - : : +- * ColumnarToRow (75) - : : +- Scan parquet spark_catalog.default.web_sales (74) - : +- ReusedExchange (80) - +- BroadcastExchange (102) - +- * HashAggregate (101) - +- Exchange (100) - +- * HashAggregate (99) - +- * Project (98) - +- * BroadcastHashJoin Inner BuildRight (97) - :- * Project (95) - : +- * BroadcastHashJoin Inner BuildRight (94) - : :- * Filter (92) - : : +- * ColumnarToRow (91) - : : +- Scan parquet spark_catalog.default.customer (90) - : +- ReusedExchange (93) - +- ReusedExchange (96) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) + : : : : :- CometFilter (17) + : : : : : +- CometHashAggregate (16) + : : : : : +- CometExchange (15) + : : : : : +- CometHashAggregate (14) + : : : : : +- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometProject (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometNativeScan parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometNativeScan parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#14)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +(5) CometFilter +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#15) -(6) Filter [codegen id : 1] -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) +(6) CometBroadcastExchange +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -(7) BroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None +(8) CometProject +Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -(9) Project [codegen id : 3] -Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#15, d_year#16] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] - -(16) HashAggregate [codegen id : 3] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [2]: [sum#17, isEmpty#18] -Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] - -(17) Exchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 24] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21] -Results [2]: [c_customer_id#2 AS customer_id#22, sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21 AS year_total#23] - -(19) Filter [codegen id : 24] -Input [2]: [customer_id#22, year_total#23] -Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.000000)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +(10) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] + +(12) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] + +(14) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(15) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(17) CometFilter +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +(19) CometFilter +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_customer_id#25)) +(20) CometProject +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#29, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#30, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#31, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#33, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#34, 50, true, false, true) AS c_email_address#14] -(23) ReusedExchange [Reuses operator id: 7] -Output [6]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#24] -Right keys [1]: [ss_customer_sk#32] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_sk#27], [ss_customer_sk#35], Inner, BuildRight -(25) Project [codegen id : 6] -Output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#41, d_year#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#38, d_year#39] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Input [14]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37, d_date_sk#38, d_year#39] - -(32) HashAggregate [codegen id : 6] -Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [partial_sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] - -(33) Exchange -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Arguments: hashpartitioning(c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21] -Results [8]: [c_customer_id#25 AS customer_id#44, c_first_name#26 AS customer_first_name#45, c_last_name#27 AS customer_last_name#46, c_preferred_cust_flag#28 AS customer_preferred_cust_flag#47, c_birth_country#29 AS customer_birth_country#48, c_login#30 AS customer_login#49, c_email_address#31 AS customer_email_address#50, sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21 AS year_total#51] - -(35) BroadcastExchange -Input [8]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#44] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] +(25) CometFilter +Input [2]: [d_date_sk#41, d_year#42] +Condition : ((isnotnull(d_year#42) AND (d_year#42 = 2002)) AND isnotnull(d_date_sk#41)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#41, d_year#42] +Arguments: [d_date_sk#41, d_year#42] + +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Right output [2]: [d_date_sk#41, d_year#42] +Arguments: [ss_sold_date_sk#40], [d_date_sk#41], Inner, BuildRight + +(28) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40, d_date_sk#41, d_year#42] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] + +(29) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [partial_sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(30) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(32) CometBroadcastExchange +Input [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#25], [customer_id#45], Inner, BuildRight + +(34) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] +(35) CometFilter +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Condition : (isnotnull(c_customer_sk#53) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true))) -(39) Filter [codegen id : 10] -Input [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] -Condition : (isnotnull(c_customer_sk#52) AND isnotnull(c_customer_id#53)) +(36) CometProject +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#55, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#56, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#57, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#59, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#60, 50, true, false, true) AS c_email_address#66] -(40) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] +(37) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#65)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#72)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] - -(42) Filter [codegen id : 8] -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Condition : isnotnull(cs_bill_customer_sk#60) - -(43) BroadcastExchange -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#52] -Right keys [1]: [cs_bill_customer_sk#60] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Input [14]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#66, d_year#67] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, d_year#67] -Input [14]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65, d_date_sk#66, d_year#67] - -(49) HashAggregate [codegen id : 10] -Input [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, d_year#67] -Keys [8]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67] -Functions [1]: [partial_sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))] -Aggregate Attributes [2]: [sum#68, isEmpty#69] -Results [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] - -(50) Exchange -Input [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] -Arguments: hashpartitioning(c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] -Keys [8]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67] -Functions [1]: [sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))#72] -Results [2]: [c_customer_id#53 AS customer_id#73, sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))#72 AS year_total#74] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#73, year_total#74] -Condition : (isnotnull(year_total#74) AND (year_total#74 > 0.000000)) - -(53) BroadcastExchange -Input [2]: [customer_id#73, year_total#74] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#73] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 24] -Output [11]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#74] -Input [12]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, customer_id#73, year_total#74] - -(56) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +(38) CometFilter +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Condition : isnotnull(cs_bill_customer_sk#67) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_sk#53], [cs_bill_customer_sk#67], Inner, BuildRight + +(41) CometProject +Input [14]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#73, d_year#74] + +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight + +(44) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72, d_date_sk#73, d_year#74] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] + +(45) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [partial_sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(46) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(48) CometFilter +Input [2]: [customer_id#77, year_total#78] +Condition : (isnotnull(year_total#78) AND (year_total#78 > 0.000000)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#77, year_total#78] + +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Right output [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#25], [customer_id#77], Inner, BuildRight + +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, customer_id#77, year_total#78] +Arguments: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78], [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] - -(58) Filter [codegen id : 14] -Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] -Condition : (isnotnull(c_customer_sk#75) AND isnotnull(c_customer_id#76)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [6]: [cs_bill_customer_sk#83, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#75] -Right keys [1]: [cs_bill_customer_sk#83] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] -Input [14]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_bill_customer_sk#83, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#89, d_year#90] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [cs_sold_date_sk#88] -Right keys [1]: [d_date_sk#89] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, d_year#90] -Input [14]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88, d_date_sk#89, d_year#90] - -(65) HashAggregate [codegen id : 14] -Input [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, d_year#90] -Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90] -Functions [1]: [partial_sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))] -Aggregate Attributes [2]: [sum#91, isEmpty#92] -Results [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] - -(66) Exchange -Input [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] -Arguments: hashpartitioning(c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] -Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90] -Functions [1]: [sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))#72] -Results [2]: [c_customer_id#76 AS customer_id#95, sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))#72 AS year_total#96] - -(68) BroadcastExchange -Input [2]: [customer_id#95, year_total#96] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#95] -Join type: Inner -Join condition: (CASE WHEN (year_total#74 > 0.000000) THEN (year_total#96 / year_total#74) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#51 / year_total#23) END) - -(70) Project [codegen id : 24] -Output [10]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96] -Input [13]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#74, customer_id#95, year_total#96] - -(71) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] +(53) CometFilter +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Condition : (isnotnull(c_customer_sk#79) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true))) + +(54) CometProject +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Arguments: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66], [c_customer_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#81, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#82, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#83, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#85, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#86, 50, true, false, true) AS c_email_address#66] + +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] + +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_sk#79], [cs_bill_customer_sk#87], Inner, BuildRight + +(57) CometProject +Input [14]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#93, d_year#94] + +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [cs_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight + +(60) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] + +(61) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [partial_sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] + +(62) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#97, year_total#98] + +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] +Right output [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#25], [customer_id#97], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#52 / year_total#26) END), BuildRight + +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78, customer_id#97, year_total#98] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] + +(67) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 18] -Input [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] +(68) CometFilter +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Condition : (isnotnull(c_customer_sk#99) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true))) -(73) Filter [codegen id : 18] -Input [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] -Condition : (isnotnull(c_customer_sk#97) AND isnotnull(c_customer_id#98)) +(69) CometProject +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Arguments: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112], [c_customer_sk#99, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#101, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#102, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#103, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#104, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#105, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#106, 50, true, false, true) AS c_email_address#112] -(74) Scan parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] +(70) CometNativeScan parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#110)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#118)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 16] -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] - -(76) Filter [codegen id : 16] -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Condition : isnotnull(ws_bill_customer_sk#105) - -(77) BroadcastExchange -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(78) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_customer_sk#97] -Right keys [1]: [ws_bill_customer_sk#105] -Join type: Inner -Join condition: None - -(79) Project [codegen id : 18] -Output [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Input [14]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] - -(80) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#111, d_year#112] - -(81) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_sold_date_sk#110] -Right keys [1]: [d_date_sk#111] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 18] -Output [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, d_year#112] -Input [14]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110, d_date_sk#111, d_year#112] - -(83) HashAggregate [codegen id : 18] -Input [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, d_year#112] -Keys [8]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112] -Functions [1]: [partial_sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))] -Aggregate Attributes [2]: [sum#113, isEmpty#114] -Results [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] - -(84) Exchange -Input [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] -Arguments: hashpartitioning(c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(85) HashAggregate [codegen id : 19] -Input [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] -Keys [8]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112] -Functions [1]: [sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))] -Aggregate Attributes [1]: [sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))#117] -Results [2]: [c_customer_id#98 AS customer_id#118, sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))#117 AS year_total#119] - -(86) Filter [codegen id : 19] -Input [2]: [customer_id#118, year_total#119] -Condition : (isnotnull(year_total#119) AND (year_total#119 > 0.000000)) - -(87) BroadcastExchange -Input [2]: [customer_id#118, year_total#119] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=14] - -(88) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#118] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 24] -Output [11]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, year_total#119] -Input [12]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, customer_id#118, year_total#119] - -(90) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] +(71) CometFilter +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Condition : isnotnull(ws_bill_customer_sk#113) + +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] + +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_sk#99], [ws_bill_customer_sk#113], Inner, BuildRight + +(74) CometProject +Input [14]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] + +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#119, d_year#120] + +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Right output [2]: [d_date_sk#119, d_year#120] +Arguments: [ws_sold_date_sk#118], [d_date_sk#119], Inner, BuildRight + +(77) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118, d_date_sk#119, d_year#120] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] + +(78) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [partial_sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] + +(79) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(80) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] + +(81) CometFilter +Input [2]: [customer_id#123, year_total#124] +Condition : (isnotnull(year_total#124) AND (year_total#124 > 0.000000)) + +(82) CometBroadcastExchange +Input [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#123, year_total#124] + +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] +Right output [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#25], [customer_id#123], Inner, BuildRight + +(84) CometProject +Input [12]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, customer_id#123, year_total#124] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] + +(85) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(91) ColumnarToRow [codegen id : 22] -Input [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] - -(92) Filter [codegen id : 22] -Input [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] -Condition : (isnotnull(c_customer_sk#120) AND isnotnull(c_customer_id#121)) - -(93) ReusedExchange [Reuses operator id: 77] -Output [6]: [ws_bill_customer_sk#128, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] - -(94) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [c_customer_sk#120] -Right keys [1]: [ws_bill_customer_sk#128] -Join type: Inner -Join condition: None - -(95) Project [codegen id : 22] -Output [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] -Input [14]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_bill_customer_sk#128, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] - -(96) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#134, d_year#135] - -(97) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#133] -Right keys [1]: [d_date_sk#134] -Join type: Inner -Join condition: None - -(98) Project [codegen id : 22] -Output [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, d_year#135] -Input [14]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133, d_date_sk#134, d_year#135] - -(99) HashAggregate [codegen id : 22] -Input [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, d_year#135] -Keys [8]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135] -Functions [1]: [partial_sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))] -Aggregate Attributes [2]: [sum#136, isEmpty#137] -Results [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] - -(100) Exchange -Input [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] -Arguments: hashpartitioning(c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(101) HashAggregate [codegen id : 23] -Input [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] -Keys [8]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135] -Functions [1]: [sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))] -Aggregate Attributes [1]: [sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))#117] -Results [2]: [c_customer_id#121 AS customer_id#140, sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))#117 AS year_total#141] - -(102) BroadcastExchange -Input [2]: [customer_id#140, year_total#141] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=16] - -(103) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#140] -Join type: Inner -Join condition: (CASE WHEN (year_total#74 > 0.000000) THEN (year_total#96 / year_total#74) END > CASE WHEN (year_total#119 > 0.000000) THEN (year_total#141 / year_total#119) END) - -(104) Project [codegen id : 24] -Output [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Input [13]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, year_total#119, customer_id#140, year_total#141] - -(105) TakeOrderedAndProject -Input [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Arguments: 100, [customer_id#44 ASC NULLS FIRST, customer_first_name#45 ASC NULLS FIRST, customer_last_name#46 ASC NULLS FIRST, customer_preferred_cust_flag#47 ASC NULLS FIRST, customer_birth_country#48 ASC NULLS FIRST, customer_login#49 ASC NULLS FIRST, customer_email_address#50 ASC NULLS FIRST], [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] +(86) CometFilter +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Condition : (isnotnull(c_customer_sk#125) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true))) + +(87) CometProject +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Arguments: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112], [c_customer_sk#125, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#127, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#128, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#129, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#131, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#132, 50, true, false, true) AS c_email_address#112] + +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] + +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_sk#125], [ws_bill_customer_sk#133], Inner, BuildRight + +(90) CometProject +Input [14]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] + +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#139, d_year#140] + +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Right output [2]: [d_date_sk#139, d_year#140] +Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight + +(93) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138, d_date_sk#139, d_year#140] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] + +(94) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [partial_sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(95) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(96) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(97) CometBroadcastExchange +Input [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#143, year_total#144] + +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] +Right output [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#25], [customer_id#143], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#124 > 0.000000) THEN (year_total#144 / year_total#124) END), BuildRight + +(99) CometProject +Input [13]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124, customer_id#143, year_total#144] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#45 ASC NULLS FIRST,customer_first_name#46 ASC NULLS FIRST,customer_last_name#47 ASC NULLS FIRST,customer_preferred_cust_flag#48 ASC NULLS FIRST,customer_birth_country#49 ASC NULLS FIRST,customer_login#50 ASC NULLS FIRST,customer_email_address#51 ASC NULLS FIRST], output=[customer_id#45,customer_first_name#46,customer_last_name#47,customer_preferred_cust_flag#48,customer_birth_country#49,customer_login#50,customer_email_address#51]), [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], 100, 0, [customer_id#45 ASC NULLS FIRST, customer_first_name#46 ASC NULLS FIRST, customer_last_name#47 ASC NULLS FIRST, customer_preferred_cust_flag#48 ASC NULLS FIRST, customer_birth_country#49 ASC NULLS FIRST, customer_login#50 ASC NULLS FIRST, customer_email_address#51 ASC NULLS FIRST], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/extended.txt new file mode 100644 index 0000000000..7b0b7d1cc0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/extended.txt @@ -0,0 +1,117 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 114 out of 114 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/simplified.txt index 1b4ee994f6..f837111fe3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/simplified.txt @@ -1,156 +1,103 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - WholeStageCodegen (24) - Project [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (19) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 - WholeStageCodegen (18) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (16) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (23) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 - WholeStageCodegen (22) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 + CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/explain.txt index 32718e056b..572c9d73af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/explain.txt @@ -1,208 +1,189 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (26) - : +- * BroadcastHashJoin Inner BuildRight (25) - : :- * Project (19) - : : +- * BroadcastHashJoin Inner BuildRight (18) - : : :- * Project (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.catalog_returns (6) - : : +- BroadcastExchange (17) - : : +- * Filter (16) - : : +- * ColumnarToRow (15) - : : +- Scan parquet spark_catalog.default.warehouse (14) - : +- BroadcastExchange (24) - : +- * Project (23) - : +- * Filter (22) - : +- * ColumnarToRow (21) - : +- Scan parquet spark_catalog.default.item (20) - +- BroadcastExchange (30) - +- * Filter (29) - +- * ColumnarToRow (28) - +- Scan parquet spark_catalog.default.date_dim (27) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.warehouse (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (26) + +- CometFilter (25) + +- CometNativeScan parquet spark_catalog.default.date_dim (24) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) -(4) Exchange +(3) CometExchange Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST], false, 0 +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.catalog_returns +(5) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) -(9) Project [codegen id : 3] -Output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +(7) CometProject Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -(10) Exchange +(8) CometExchange Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_order_number#3, cs_item_sk#2] -Right keys [2]: [cr_order_number#7, cr_item_sk#6] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter -(13) Project [codegen id : 8] -Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +(11) CometProject Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -(14) Scan parquet spark_catalog.default.warehouse +(12) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#10, w_state#11] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [2]: [w_warehouse_sk#10, w_state#11] - -(16) Filter [codegen id : 5] +(13) CometFilter Input [2]: [w_warehouse_sk#10, w_state#11] Condition : isnotnull(w_warehouse_sk#10) -(17) BroadcastExchange +(14) CometProject Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#11, 2, true, false, true) AS w_state#12] -(18) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_warehouse_sk#1] -Right keys [1]: [w_warehouse_sk#10] -Join type: Inner -Join condition: None +(15) CometBroadcastExchange +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] -(19) Project [codegen id : 8] -Output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] +(16) CometBroadcastHashJoin +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +Right output [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight -(20) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] +(17) CometProject +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] + +(18) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(22) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) +(19) CometFilter +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) -(23) Project [codegen id : 6] -Output [2]: [i_item_sk#12, i_item_id#13] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] +(20) CometProject +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#14, 16, true, false, true) AS i_item_id#16] -(24) BroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] -(25) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight -(26) Project [codegen id : 8] -Output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] +(23) CometProject +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] -(27) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_date#16] +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#17, d_date#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#15, d_date#16] - -(29) Filter [codegen id : 7] -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) - -(30) BroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] - -(33) HashAggregate [codegen id : 8] -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [4]: [sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] - -(34) Exchange -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) HashAggregate [codegen id : 9] -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26] -Results [4]: [w_state#11, i_item_id#13, sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25 AS sales_before#27, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26 AS sales_after#28] - -(36) TakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#27, sales_after#28] -Arguments: 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#27, sales_after#28] +(25) CometFilter +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] + +(27) CometBroadcastHashJoin +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight + +(28) CometProject +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] + +(29) CometHashAggregate +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(30) CometExchange +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(31) CometHashAggregate +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(32) CometTakeOrderedAndProject +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] + +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/extended.txt new file mode 100644 index 0000000000..7a22197a97 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.warehouse + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/simplified.txt index 6645ee24b7..98932cc775 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/simplified.txt @@ -1,58 +1,35 @@ -TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - WholeStageCodegen (9) - HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_state,i_item_id] #1 - WholeStageCodegen (8) - HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Project [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #2 - WholeStageCodegen (1) - Filter [cs_warehouse_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #3 - WholeStageCodegen (3) - Project [cr_item_sk,cr_order_number,cr_refunded_cash] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [w_state,i_item_id,sales_before,sales_after,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] + CometExchange [w_state,i_item_id] #1 + CometHashAggregate [d_date,cs_sales_price,cr_refunded_cash] [w_state,i_item_id,sum,isEmpty,sum,isEmpty] + CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] + CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] + CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] + CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] + CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] + CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #2 + CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] + CometExchange [cr_order_number,cr_item_sk] #3 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_state] #4 + CometProject [w_state] [w_warehouse_sk,w_state] + CometFilter [w_warehouse_sk,w_state] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/explain.txt index 71cad8f2f0..cbce1b2b45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/explain.txt @@ -1,121 +1,107 @@ == Physical Plan == -TakeOrderedAndProject (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (4) - : +- * Filter (3) - : +- * ColumnarToRow (2) - : +- Scan parquet spark_catalog.default.item (1) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) - +- Scan parquet spark_catalog.default.item (5) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (3) + : +- CometFilter (2) + : +- CometNativeScan parquet spark_catalog.default.item (1) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometHashAggregate (9) + +- CometExchange (8) + +- CometHashAggregate (7) + +- CometProject (6) + +- CometFilter (5) + +- CometNativeScan parquet spark_catalog.default.item (4) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)] +PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) -(3) Filter [codegen id : 3] +(3) CometProject Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) +Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) Project [codegen id : 3] -Output [2]: [i_manufact#2, i_product_name#3] -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] - -(5) Scan parquet spark_catalog.default.item -Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +(4) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,powder ),EqualTo(i_color,khaki )),Or(EqualTo(i_units,Ounce ),EqualTo(i_units,Oz ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,brown ),EqualTo(i_color,honeydew )),Or(EqualTo(i_units,Bunch ),EqualTo(i_units,Ton ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,floral ),EqualTo(i_color,deep )),Or(EqualTo(i_units,N/A ),EqualTo(i_units,Dozen ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,light ),EqualTo(i_color,cornflower )),Or(EqualTo(i_units,Box ),EqualTo(i_units,Pound ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large )))))),Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,midnight ),EqualTo(i_color,snow )),Or(EqualTo(i_units,Pallet ),EqualTo(i_units,Gross ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,cyan ),EqualTo(i_color,papaya )),Or(EqualTo(i_units,Cup ),EqualTo(i_units,Dram ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,orange ),EqualTo(i_color,frosted )),Or(EqualTo(i_units,Each ),EqualTo(i_units,Tbl ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,forest ),EqualTo(i_color,ghost )),Or(EqualTo(i_units,Lb ),EqualTo(i_units,Bundle ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))))))), IsNotNull(i_manufact)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] - -(7) Filter [codegen id : 1] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) +(5) CometFilter +Input [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] +Condition : (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = powder ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = khaki )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Ounce ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Oz ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = brown ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = honeydew )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Bunch ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Ton ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = floral ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = deep )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Dozen ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = light ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = cornflower )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Box ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Pound ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large )))))) OR (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = midnight ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = snow )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Pallet ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Gross ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = cyan ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = papaya )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Cup ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Dram ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = orange ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = frosted )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Each ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Tbl ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = forest ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = ghost )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Lb ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Bundle ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large ))))))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true))) -(8) Project [codegen id : 1] -Output [1]: [i_manufact#5] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +(6) CometProject +Input [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] +Arguments: [i_manufact#8], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true) AS i_manufact#8] -(9) HashAggregate [codegen id : 1] -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] +(7) CometHashAggregate +Input [1]: [i_manufact#8] +Keys [1]: [i_manufact#8] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#9] -Results [2]: [i_manufact#5, count#10] -(10) Exchange -Input [2]: [i_manufact#5, count#10] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(8) CometExchange +Input [2]: [i_manufact#8, count#9] +Arguments: hashpartitioning(i_manufact#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(11) HashAggregate [codegen id : 2] -Input [2]: [i_manufact#5, count#10] -Keys [1]: [i_manufact#5] +(9) CometHashAggregate +Input [2]: [i_manufact#8, count#9] +Keys [1]: [i_manufact#8] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#11] -Results [2]: [count(1)#11 AS item_cnt#12, i_manufact#5] - -(12) Filter [codegen id : 2] -Input [2]: [item_cnt#12, i_manufact#5] -Condition : (item_cnt#12 > 0) - -(13) Project [codegen id : 2] -Output [1]: [i_manufact#5] -Input [2]: [item_cnt#12, i_manufact#5] - -(14) BroadcastExchange -Input [1]: [i_manufact#5] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_manufact#2] -Right keys [1]: [i_manufact#5] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [1]: [i_product_name#3] -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] - -(17) HashAggregate [codegen id : 3] -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] + +(10) CometFilter +Input [2]: [item_cnt#10, i_manufact#8] +Condition : (item_cnt#10 > 0) + +(11) CometProject +Input [2]: [item_cnt#10, i_manufact#8] +Arguments: [i_manufact#8], [i_manufact#8] + +(12) CometBroadcastExchange +Input [1]: [i_manufact#8] +Arguments: [i_manufact#8] + +(13) CometBroadcastHashJoin +Left output [2]: [i_manufact#2, i_product_name#3] +Right output [1]: [i_manufact#8] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true)], [i_manufact#8], Inner, BuildRight + +(14) CometProject +Input [3]: [i_manufact#2, i_product_name#3, i_manufact#8] +Arguments: [i_product_name#11], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#3, 50, true, false, true) AS i_product_name#11] + +(15) CometHashAggregate +Input [1]: [i_product_name#11] +Keys [1]: [i_product_name#11] Functions: [] -Aggregate Attributes: [] -Results [1]: [i_product_name#3] -(18) Exchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [1]: [i_product_name#11] +Arguments: hashpartitioning(i_product_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(19) HashAggregate [codegen id : 4] -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +(17) CometHashAggregate +Input [1]: [i_product_name#11] +Keys [1]: [i_product_name#11] Functions: [] -Aggregate Attributes: [] -Results [1]: [i_product_name#3] -(20) TakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] +(18) CometTakeOrderedAndProject +Input [1]: [i_product_name#11] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#11 ASC NULLS FIRST], output=[i_product_name#11]), [i_product_name#11], 100, 0, [i_product_name#11 ASC NULLS FIRST], [i_product_name#11] + +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [i_product_name#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/extended.txt new file mode 100644 index 0000000000..b40ebddb6f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/simplified.txt index e287c0fd05..2557f51684 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/simplified.txt @@ -1,29 +1,21 @@ -TakeOrderedAndProject [i_product_name] - WholeStageCodegen (4) - HashAggregate [i_product_name] - InputAdapter - Exchange [i_product_name] #1 - WholeStageCodegen (3) - HashAggregate [i_product_name] - Project [i_product_name] - BroadcastHashJoin [i_manufact,i_manufact] - Project [i_manufact,i_product_name] - Filter [i_manufact_id,i_manufact] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [i_manufact] - Filter [item_cnt] - HashAggregate [i_manufact,count] [count(1),item_cnt,count] - InputAdapter - Exchange [i_manufact] #3 - WholeStageCodegen (1) - HashAggregate [i_manufact] [count,count] - Project [i_manufact] - Filter [i_category,i_color,i_units,i_size,i_manufact] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name] + CometHashAggregate [i_product_name] + CometExchange [i_product_name] #1 + CometHashAggregate [i_product_name] + CometProject [i_product_name] [i_product_name] + CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] + CometProject [i_manufact,i_product_name] + CometFilter [i_manufact_id,i_manufact,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometBroadcastExchange [i_manufact] #2 + CometProject [i_manufact] + CometFilter [item_cnt,i_manufact] + CometHashAggregate [count] [item_cnt,i_manufact,count(1)] + CometExchange [i_manufact] #3 + CometHashAggregate [i_manufact,count] + CometProject [i_manufact] [i_manufact] + CometFilter [i_category,i_manufact,i_size,i_color,i_units] + CometNativeScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/explain.txt index 327c17b215..d0f741f459 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/explain.txt @@ -1,125 +1,111 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.item (9) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_category_id#8, i_category#9] +(11) CometProject Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11], [i_item_sk#7, i_category_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -(19) Exchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +(17) CometHashAggregate +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_category_id#8, i_category#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] -(21) TakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] -Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] +(18) CometTakeOrderedAndProject +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sum(ss_ext_sales_price)#13 DESC NULLS LAST,d_year#2 ASC NULLS FIRST,i_category_id#8 ASC NULLS FIRST,i_category#11 ASC NULLS FIRST], output=[d_year#2,i_category_id#8,i_category#11,sum(ss_ext_sales_price)#13]), [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13], 100, 0, [sum(ss_ext_sales_price)#13 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#11 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] + +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/simplified.txt index aaf2b9dc2f..96ab34454d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] - WholeStageCodegen (4) - HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum] - InputAdapter - Exchange [d_year,i_category_id,i_category] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_category_id,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_category_id,i_category] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [d_year,i_category_id,i_category,sum(ss_ext_sales_price)] + CometHashAggregate [sum] [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [d_year,i_category_id,i_category] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_category_id,i_category,sum] + CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 + CometProject [i_category] [i_item_sk,i_category_id,i_category] + CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/explain.txt index b21f03173d..807307f250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/explain.txt @@ -1,125 +1,111 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.store (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.store (9) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_day_name#3] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] +Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(5) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) +(5) CometFilter +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_store_sk#5) -(8) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_day_name#4] +Right output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +(8) CometProject +Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(11) Scan parquet spark_catalog.default.store -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] +(9) CometNativeScan parquet spark_catalog.default.store +Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(13) Filter [codegen id : 2] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) - -(14) Project [codegen id : 2] -Output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(15) BroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 3] -Output [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] - -(18) HashAggregate [codegen id : 3] -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Results [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] - -(19) Exchange -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31] -Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] - -(21) TakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] -Arguments: 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#32 ASC NULLS FIRST, mon_sales#33 ASC NULLS FIRST, tue_sales#34 ASC NULLS FIRST, wed_sales#35 ASC NULLS FIRST, thu_sales#36 ASC NULLS FIRST, fri_sales#37 ASC NULLS FIRST, sat_sales#38 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] +(10) CometFilter +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8)) + +(11) CometProject +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10], [s_store_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#9, 16, true, false, true) AS s_store_id#12, s_store_name#10] + +(12) CometBroadcastExchange +Input [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10] + +(13) CometBroadcastHashJoin +Left output [3]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6] +Right output [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [ss_store_sk#5], [s_store_sk#8], Inner, BuildRight + +(14) CometProject +Input [6]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6, s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10], [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] + +(15) CometHashAggregate +Input [4]: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] + +(16) CometExchange +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(s_store_name#10, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometHashAggregate +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] + +(18) CometTakeOrderedAndProject +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#10 ASC NULLS FIRST,s_store_id#12 ASC NULLS FIRST,sun_sales#20 ASC NULLS FIRST,mon_sales#21 ASC NULLS FIRST,tue_sales#22 ASC NULLS FIRST,wed_sales#23 ASC NULLS FIRST,thu_sales#24 ASC NULLS FIRST,fri_sales#25 ASC NULLS FIRST,sat_sales#26 ASC NULLS FIRST], output=[s_store_name#10,s_store_id#12,sun_sales#20,mon_sales#21,tue_sales#22,wed_sales#23,thu_sales#24,fri_sales#25,sat_sales#26]), [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26], 100, 0, [s_store_name#10 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sun_sales#20 ASC NULLS FIRST, mon_sales#21 ASC NULLS FIRST, tue_sales#22 ASC NULLS FIRST, wed_sales#23 ASC NULLS FIRST, thu_sales#24 ASC NULLS FIRST, fri_sales#25 ASC NULLS FIRST, sat_sales#26 ASC NULLS FIRST], [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] + +(19) CometColumnarToRow [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/extended.txt new file mode 100644 index 0000000000..5eaea983af --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/simplified.txt index 3e73642c4f..c7064b3ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - WholeStageCodegen (4) - HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_store_id] #1 - WholeStageCodegen (3) - HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [d_day_name,ss_sales_price,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [d_day_name,ss_store_sk,ss_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_day_name] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk,s_store_id,s_store_name] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometExchange [s_store_name,s_store_id] #1 + CometHashAggregate [d_day_name,ss_sales_price] [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] + CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] + CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] + CometProject [d_day_name,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [d_day_name] [d_date_sk,d_day_name] + CometFilter [d_date_sk,d_year,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/explain.txt index 4eda34a7e5..e66d71b41e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/explain.txt @@ -1,226 +1,227 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * BroadcastHashJoin Inner BuildRight (30) - :- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (22) - : : +- * SortMergeJoin Inner (21) +TakeOrderedAndProject (34) ++- * Project (33) + +- * BroadcastHashJoin Inner BuildRight (32) + :- * Project (30) + : +- * BroadcastHashJoin Inner BuildRight (29) + : :- * Project (23) + : : +- * SortMergeJoin Inner (22) : : :- * Sort (14) : : : +- * Project (13) : : : +- * Filter (12) : : : +- Window (11) - : : : +- * Sort (10) - : : : +- Exchange (9) - : : : +- * Filter (8) - : : : +- * HashAggregate (7) - : : : +- Exchange (6) - : : : +- * HashAggregate (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- * Sort (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- Window (17) - : : +- * Sort (16) - : : +- ReusedExchange (15) - : +- BroadcastExchange (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- ReusedExchange (29) - - -(1) Scan parquet spark_catalog.default.store_sales + : : : +- * CometColumnarToRow (10) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometFilter (7) + : : : +- CometHashAggregate (6) + : : : +- CometExchange (5) + : : : +- CometHashAggregate (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- * Sort (21) + : : +- * Project (20) + : : +- * Filter (19) + : : +- Window (18) + : : +- * CometColumnarToRow (17) + : : +- CometSort (16) + : : +- ReusedExchange (15) + : +- BroadcastExchange (28) + : +- * CometColumnarToRow (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.item (24) + +- ReusedExchange (31) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) -(4) Project [codegen id : 1] -Output [2]: [ss_item_sk#1, ss_net_profit#3] +(3) CometProject Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [ss_item_sk#1, ss_net_profit#3], [ss_item_sk#1, ss_net_profit#3] -(5) HashAggregate [codegen id : 1] +(4) CometHashAggregate Input [2]: [ss_item_sk#1, ss_net_profit#3] Keys [1]: [ss_item_sk#1] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#5, count#6] -Results [3]: [ss_item_sk#1, sum#7, count#8] -(6) Exchange -Input [3]: [ss_item_sk#1, sum#7, count#8] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(5) CometExchange +Input [3]: [ss_item_sk#1, sum#5, count#6] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(7) HashAggregate [codegen id : 2] -Input [3]: [ss_item_sk#1, sum#7, count#8] +(6) CometHashAggregate +Input [3]: [ss_item_sk#1, sum#5, count#6] Keys [1]: [ss_item_sk#1] Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#9] -Results [2]: [ss_item_sk#1 AS item_sk#10, cast((avg(UnscaledValue(ss_net_profit#3))#9 / 100.0) as decimal(11,6)) AS rank_col#11] -(8) Filter [codegen id : 2] -Input [2]: [item_sk#10, rank_col#11] -Condition : (isnotnull(rank_col#11) AND (cast(rank_col#11 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#12, [id=#13]))) +(7) CometFilter +Input [2]: [item_sk#7, rank_col#8] +Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#9, [id=#10]))) -(9) Exchange -Input [2]: [item_sk#10, rank_col#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(8) CometExchange +Input [2]: [item_sk#7, rank_col#8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(10) Sort [codegen id : 3] -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 +(9) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(10) CometColumnarToRow [codegen id : 1] +Input [2]: [item_sk#7, rank_col#8] (11) Window -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank(rank_col#11) windowspecdefinition(rank_col#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#11 ASC NULLS FIRST] +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] -(12) Filter [codegen id : 4] -Input [3]: [item_sk#10, rank_col#11, rnk#14] -Condition : ((rnk#14 < 11) AND isnotnull(item_sk#10)) +(12) Filter [codegen id : 2] +Input [3]: [item_sk#7, rank_col#8, rnk#11] +Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) -(13) Project [codegen id : 4] -Output [2]: [item_sk#10, rnk#14] -Input [3]: [item_sk#10, rank_col#11, rnk#14] +(13) Project [codegen id : 2] +Output [2]: [item_sk#7, rnk#11] +Input [3]: [item_sk#7, rank_col#8, rnk#11] -(14) Sort [codegen id : 4] -Input [2]: [item_sk#10, rnk#14] -Arguments: [rnk#14 ASC NULLS FIRST], false, 0 +(14) Sort [codegen id : 2] +Input [2]: [item_sk#7, rnk#11] +Arguments: [rnk#11 ASC NULLS FIRST], false, 0 -(15) ReusedExchange [Reuses operator id: 9] -Output [2]: [item_sk#15, rank_col#16] +(15) ReusedExchange [Reuses operator id: 8] +Output [2]: [item_sk#12, rank_col#13] -(16) Sort [codegen id : 7] -Input [2]: [item_sk#15, rank_col#16] -Arguments: [rank_col#16 DESC NULLS LAST], false, 0 +(16) CometSort +Input [2]: [item_sk#12, rank_col#13] +Arguments: [item_sk#12, rank_col#13], [rank_col#13 DESC NULLS LAST] -(17) Window -Input [2]: [item_sk#15, rank_col#16] -Arguments: [rank(rank_col#16) windowspecdefinition(rank_col#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#17], [rank_col#16 DESC NULLS LAST] +(17) CometColumnarToRow [codegen id : 3] +Input [2]: [item_sk#12, rank_col#13] -(18) Filter [codegen id : 8] -Input [3]: [item_sk#15, rank_col#16, rnk#17] -Condition : ((rnk#17 < 11) AND isnotnull(item_sk#15)) +(18) Window +Input [2]: [item_sk#12, rank_col#13] +Arguments: [rank(rank_col#13) windowspecdefinition(rank_col#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#13 DESC NULLS LAST] -(19) Project [codegen id : 8] -Output [2]: [item_sk#15, rnk#17] -Input [3]: [item_sk#15, rank_col#16, rnk#17] +(19) Filter [codegen id : 4] +Input [3]: [item_sk#12, rank_col#13, rnk#14] +Condition : ((rnk#14 < 11) AND isnotnull(item_sk#12)) -(20) Sort [codegen id : 8] -Input [2]: [item_sk#15, rnk#17] -Arguments: [rnk#17 ASC NULLS FIRST], false, 0 +(20) Project [codegen id : 4] +Output [2]: [item_sk#12, rnk#14] +Input [3]: [item_sk#12, rank_col#13, rnk#14] -(21) SortMergeJoin [codegen id : 11] -Left keys [1]: [rnk#14] -Right keys [1]: [rnk#17] +(21) Sort [codegen id : 4] +Input [2]: [item_sk#12, rnk#14] +Arguments: [rnk#14 ASC NULLS FIRST], false, 0 + +(22) SortMergeJoin [codegen id : 7] +Left keys [1]: [rnk#11] +Right keys [1]: [rnk#14] Join type: Inner Join condition: None -(22) Project [codegen id : 11] -Output [3]: [item_sk#10, rnk#14, item_sk#15] -Input [4]: [item_sk#10, rnk#14, item_sk#15, rnk#17] +(23) Project [codegen id : 7] +Output [3]: [item_sk#7, rnk#11, item_sk#12] +Input [4]: [item_sk#7, rnk#11, item_sk#12, rnk#14] -(23) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#18, i_product_name#19] +(24) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_product_name#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 9] -Input [2]: [i_item_sk#18, i_product_name#19] +(25) CometFilter +Input [2]: [i_item_sk#15, i_product_name#16] +Condition : isnotnull(i_item_sk#15) -(25) Filter [codegen id : 9] -Input [2]: [i_item_sk#18, i_product_name#19] -Condition : isnotnull(i_item_sk#18) +(26) CometProject +Input [2]: [i_item_sk#15, i_product_name#16] +Arguments: [i_item_sk#15, i_product_name#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#16, 50, true, false, true) AS i_product_name#17] -(26) BroadcastExchange -Input [2]: [i_item_sk#18, i_product_name#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(27) CometColumnarToRow [codegen id : 5] +Input [2]: [i_item_sk#15, i_product_name#17] -(27) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [item_sk#10] -Right keys [1]: [i_item_sk#18] +(28) BroadcastExchange +Input [2]: [i_item_sk#15, i_product_name#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(29) BroadcastHashJoin [codegen id : 7] +Left keys [1]: [item_sk#7] +Right keys [1]: [i_item_sk#15] Join type: Inner Join condition: None -(28) Project [codegen id : 11] -Output [3]: [rnk#14, item_sk#15, i_product_name#19] -Input [5]: [item_sk#10, rnk#14, item_sk#15, i_item_sk#18, i_product_name#19] +(30) Project [codegen id : 7] +Output [3]: [rnk#11, item_sk#12, i_product_name#17] +Input [5]: [item_sk#7, rnk#11, item_sk#12, i_item_sk#15, i_product_name#17] -(29) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_item_sk#20, i_product_name#21] +(31) ReusedExchange [Reuses operator id: 28] +Output [2]: [i_item_sk#18, i_product_name#19] -(30) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [item_sk#15] -Right keys [1]: [i_item_sk#20] +(32) BroadcastHashJoin [codegen id : 7] +Left keys [1]: [item_sk#12] +Right keys [1]: [i_item_sk#18] Join type: Inner Join condition: None -(31) Project [codegen id : 11] -Output [3]: [rnk#14, i_product_name#19 AS best_performing#22, i_product_name#21 AS worst_performing#23] -Input [5]: [rnk#14, item_sk#15, i_product_name#19, i_item_sk#20, i_product_name#21] +(33) Project [codegen id : 7] +Output [3]: [rnk#11, i_product_name#17 AS best_performing#20, i_product_name#19 AS worst_performing#21] +Input [5]: [rnk#11, item_sk#12, i_product_name#17, i_item_sk#18, i_product_name#19] -(32) TakeOrderedAndProject -Input [3]: [rnk#14, best_performing#22, worst_performing#23] -Arguments: 100, [rnk#14 ASC NULLS FIRST], [rnk#14, best_performing#22, worst_performing#23] +(34) TakeOrderedAndProject +Input [3]: [rnk#11, best_performing#20, worst_performing#21] +Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#20, worst_performing#21] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* HashAggregate (39) -+- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * Filter (35) - +- * ColumnarToRow (34) - +- Scan parquet spark_catalog.default.store_sales (33) +Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometFilter (36) + +- CometNativeScan parquet spark_catalog.default.store_sales (35) -(33) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] +(35) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 1] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] - -(35) Filter [codegen id : 1] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] -Condition : ((isnotnull(ss_store_sk#25) AND (ss_store_sk#25 = 4)) AND isnull(ss_addr_sk#24)) - -(36) Project [codegen id : 1] -Output [2]: [ss_store_sk#25, ss_net_profit#26] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] - -(37) HashAggregate [codegen id : 1] -Input [2]: [ss_store_sk#25, ss_net_profit#26] -Keys [1]: [ss_store_sk#25] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#26))] -Aggregate Attributes [2]: [sum#28, count#29] -Results [3]: [ss_store_sk#25, sum#30, count#31] - -(38) Exchange -Input [3]: [ss_store_sk#25, sum#30, count#31] -Arguments: hashpartitioning(ss_store_sk#25, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(39) HashAggregate [codegen id : 2] -Input [3]: [ss_store_sk#25, sum#30, count#31] -Keys [1]: [ss_store_sk#25] -Functions [1]: [avg(UnscaledValue(ss_net_profit#26))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#26))#32] -Results [1]: [cast((avg(UnscaledValue(ss_net_profit#26))#32 / 100.0) as decimal(11,6)) AS rank_col#33] +(36) CometFilter +Input [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] +Condition : ((isnotnull(ss_store_sk#23) AND (ss_store_sk#23 = 4)) AND isnull(ss_addr_sk#22)) + +(37) CometProject +Input [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] +Arguments: [ss_store_sk#23, ss_net_profit#24], [ss_store_sk#23, ss_net_profit#24] + +(38) CometHashAggregate +Input [2]: [ss_store_sk#23, ss_net_profit#24] +Keys [1]: [ss_store_sk#23] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#24))] + +(39) CometExchange +Input [3]: [ss_store_sk#23, sum#26, count#27] +Arguments: hashpartitioning(ss_store_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(40) CometHashAggregate +Input [3]: [ss_store_sk#23, sum#26, count#27] +Keys [1]: [ss_store_sk#23] +Functions [1]: [avg(UnscaledValue(ss_net_profit#24))] + +(41) CometColumnarToRow [codegen id : 1] +Input [1]: [rank_col#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/extended.txt new file mode 100644 index 0000000000..cd56cd1676 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/extended.txt @@ -0,0 +1,63 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- SortMergeJoin + : : :- Sort + : : : +- Project + : : : +- Filter + : : : +- Window + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : : +- Subquery + : : : : +- CometColumnarToRow + : : : : +- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- Sort + : : +- Project + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 36 out of 55 eligible operators (65%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/simplified.txt index 46d14b650b..6ff27fc6c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] - WholeStageCodegen (11) + WholeStageCodegen (7) Project [rnk,i_product_name,i_product_name] BroadcastHashJoin [item_sk,i_item_sk] Project [rnk,item_sk,i_product_name] @@ -7,57 +7,53 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Project [item_sk,rnk,item_sk] SortMergeJoin [rnk,rnk] InputAdapter - WholeStageCodegen (4) + WholeStageCodegen (2) Sort [rnk] Project [item_sk,rnk] Filter [rnk,item_sk] InputAdapter Window [rank_col] - WholeStageCodegen (3) - Sort [rank_col] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (2) - Filter [rank_col] + CometSort [item_sk,rank_col] + CometExchange #1 + CometFilter [item_sk,rank_col] Subquery #1 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [ss_store_sk] #3 - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_store_sk,ss_net_profit] - Filter [ss_store_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - InputAdapter - Exchange [ss_item_sk] #2 - WholeStageCodegen (1) - HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_item_sk,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [ss_store_sk,sum,count] [rank_col,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_store_sk] #3 + CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_item_sk] #2 + CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter - WholeStageCodegen (8) + WholeStageCodegen (4) Sort [rnk] Project [item_sk,rnk] Filter [rnk,item_sk] InputAdapter Window [rank_col] - WholeStageCodegen (7) - Sort [rank_col] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - ReusedExchange [item_sk,rank_col] #1 + CometSort [item_sk,rank_col] + ReusedExchange [item_sk,rank_col] #1 InputAdapter BroadcastExchange #4 - WholeStageCodegen (9) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_product_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/explain.txt index 0350ba69bd..f2a6d50865 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/explain.txt @@ -1,232 +1,226 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * Filter (35) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (34) - :- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.date_dim (16) - : +- BroadcastExchange (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.item (29) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (39) ++- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * Project (34) + +- * Filter (33) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (32) + :- * CometColumnarToRow (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- BroadcastExchange (31) + +- * CometColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) + +- CometNativeScan parquet spark_catalog.default.item (27) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#5)] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) -(4) Scan parquet spark_catalog.default.customer +(3) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#6, c_current_addr_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [c_customer_sk#6, c_current_addr_sk#7] Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [c_customer_sk#6, c_current_addr_sk#7] -(8) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_bill_customer_sk#3] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight -(9) Project [codegen id : 6] -Output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +(7) CometProject Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Condition : isnotnull(ca_address_sk#8) -(13) BroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#10, 10, true, false, true) AS ca_zip#11] -(14) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_current_addr_sk#7] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] + +(12) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight -(15) Project [codegen id : 6] -Output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] +(13) CometProject +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +(15) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) +(16) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#11] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] -(20) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#5] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight -(22) Project [codegen id : 6] -Output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] +(19) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] -(23) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#14, i_item_id#15] +(20) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#14, i_item_id#15] +(21) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) -(25) Filter [codegen id : 4] -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) +(22) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] -(26) BroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#2] -Right keys [1]: [i_item_sk#14] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] -(28) Project [codegen id : 6] -Output [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] -(29) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +(27) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [i_item_sk#16, i_item_id#17] +(28) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : i_item_sk#15 IN (2,3,5,7,11,13,17,19,23,29) -(31) Filter [codegen id : 5] -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) +(29) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_id#18], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#18] -(32) Project [codegen id : 5] -Output [1]: [i_item_id#17] -Input [2]: [i_item_sk#16, i_item_id#17] +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [i_item_id#18] -(33) BroadcastExchange -Input [1]: [i_item_id#17] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] +(31) BroadcastExchange +Input [1]: [i_item_id#18] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] +(32) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#18] Join type: ExistenceJoin(exists#1) Join condition: None -(35) Filter [codegen id : 6] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +(33) Filter [codegen id : 2] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) -(36) Project [codegen id : 6] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] +(34) Project [codegen id : 2] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] -(37) HashAggregate [codegen id : 6] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] +(35) HashAggregate [codegen id : 2] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] +Aggregate Attributes [1]: [sum#19] +Results [3]: [ca_zip#11, ca_city#9, sum#20] + +(36) CometColumnarExchange +Input [3]: [ca_zip#11, ca_city#9, sum#20] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(38) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(37) CometColumnarToRow [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#20] -(39) HashAggregate [codegen id : 7] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] +(38) HashAggregate [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#20] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#21] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#21,17,2) AS sum(ws_sales_price)#22] -(40) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] +(39) TakeOrderedAndProject +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#22] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/extended.txt new file mode 100644 index 0000000000..8fb8fa7188 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/extended.txt @@ -0,0 +1,41 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- Filter + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 29 out of 36 eligible operators (80%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/simplified.txt index a36bd2c3ca..fd8a442c1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/simplified.txt @@ -1,59 +1,46 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] - WholeStageCodegen (7) + WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (6) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - Project [ws_sales_price,ca_city,ca_zip,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_sales_price,ca_city,ca_zip] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Filter [ws_bill_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 + CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/explain.txt index 245d1a3d5b..944bb79698 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/explain.txt @@ -1,248 +1,223 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Project (24) - : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.store (11) - : : : +- BroadcastExchange (22) - : : : +- * Project (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.household_demographics (18) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.customer_address (25) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.customer (34) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometHashAggregate (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometNativeScan parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(5) CometProject Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(8) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_city#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_city#13] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#12, s_city#13] Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#12] +(11) CometProject Input [2]: [s_store_sk#12, s_city#13] +Arguments: [s_store_sk#12], [s_store_sk#12] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#12] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [1]: [s_store_sk#12] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(14) CometProject Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#14] +(17) CometProject Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#14] -(23) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#14] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [1]: [hd_demo_sk#14] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight -(24) Project [codegen id : 5] -Output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(20) CometProject Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(25) Scan parquet spark_catalog.default.customer_address +(21) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_city#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_city#18] - -(27) Filter [codegen id : 4] +(22) CometFilter Input [2]: [ca_address_sk#17, ca_city#18] Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) -(28) BroadcastExchange +(23) CometBroadcastExchange Input [2]: [ca_address_sk#17, ca_city#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [ca_address_sk#17, ca_city#18] -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight -(30) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] +(25) CometProject Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] -(31) HashAggregate [codegen id : 5] +(26) CometHashAggregate Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#19, sum#20] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -(32) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 8] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] +(28) CometHashAggregate +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#23, sum(UnscaledValue(ss_net_profit#7))#24] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#25, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#23,17,2) AS amt#26, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#24,17,2) AS profit#27] -(34) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(29) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(30) CometFilter +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) + +(31) CometProject +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [c_customer_sk#21, c_current_addr_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#26] -(36) Filter [codegen id : 6] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_current_addr_sk#29)) +(32) CometBroadcastExchange +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] -(37) BroadcastExchange -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(33) CometBroadcastHashJoin +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29] +Right output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#28] -Join type: Inner -Join condition: None +(34) CometProject +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29, c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] -(39) Project [codegen id : 8] -Output [7]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27, c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(35) ReusedExchange [Reuses operator id: 23] +Output [2]: [ca_address_sk#30, ca_city#31] -(40) ReusedExchange [Reuses operator id: 28] -Output [2]: [ca_address_sk#32, ca_city#33] +(36) CometBroadcastHashJoin +Left output [7]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Right output [2]: [ca_address_sk#30, ca_city#31] +Arguments: [c_current_addr_sk#22], [ca_address_sk#30], Inner, NOT (ca_city#31 = bought_city#27), BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [c_current_addr_sk#29] -Right keys [1]: [ca_address_sk#32] -Join type: Inner -Join condition: NOT (ca_city#33 = bought_city#25) +(37) CometProject +Input [9]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26, ca_address_sk#30, ca_city#31] +Arguments: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] -(42) Project [codegen id : 8] -Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Input [9]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] +(38) CometTakeOrderedAndProject +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#26 ASC NULLS FIRST,c_first_name#25 ASC NULLS FIRST,ca_city#31 ASC NULLS FIRST,bought_city#27 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#26,c_first_name#25,ca_city#31,bought_city#27,ss_ticket_number#5,amt#28,profit#29]), [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], 100, 0, [c_last_name#26 ASC NULLS FIRST, c_first_name#25 ASC NULLS FIRST, ca_city#31 ASC NULLS FIRST, bought_city#27 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] -(43) TakeOrderedAndProject -Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#25 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] +(39) CometColumnarToRow [codegen id : 1] +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/extended.txt new file mode 100644 index 0000000000..2cc4ab6ef2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 40 out of 40 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/simplified.txt index a6a56fe693..11f3dc34da 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/simplified.txt @@ -1,63 +1,41 @@ -TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - WholeStageCodegen (8) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (5) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ca_address_sk,ca_city] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] + CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] + CometBroadcastHashJoin [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] + CometProject [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometHashAggregate [ss_addr_sk,sum,sum] [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,ca_city,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/explain.txt index fcd2b4779d..b0e9f9ad7a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.store (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#4] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(14) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) -(18) Filter [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] -(19) BroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#5] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] -(21) Project [codegen id : 4] -Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] -(27) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] -(30) Filter [codegen id : 22] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -(32) ReusedExchange [Reuses operator id: 23] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(33) HashAggregate [codegen id : 12] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#17] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(36) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#17 AS sum_sales#30, rn#29] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17, rn#29] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(38) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#29 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] + +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(41) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#17 AS sum_sales#38, rn#37] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17, rn#37] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=8] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#37 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] +(47) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/extended.txt new file mode 100644 index 0000000000..eca90ab6dc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/simplified.txt index 20a2eeef3a..61ae95b11e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/explain.txt index c0cb413c85..01cb838bef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/explain.txt @@ -1,188 +1,169 @@ == Physical Plan == -* HashAggregate (32) -+- Exchange (31) - +- * HashAggregate (30) - +- * Project (29) - +- * BroadcastHashJoin Inner BuildRight (28) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.customer_demographics (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.customer_address (16) - +- BroadcastExchange (27) - +- * Project (26) - +- * Filter (25) - +- * ColumnarToRow (24) - +- Scan parquet spark_catalog.default.date_dim (23) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (29) ++- CometHashAggregate (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.store (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.customer_address (14) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometNativeScan parquet spark_catalog.default.date_dim (20) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(4) Scan parquet spark_catalog.default.store +(3) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [s_store_sk#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [s_store_sk#8] Condition : isnotnull(s_store_sk#8) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [s_store_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [s_store_sk#8] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#8] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [1]: [s_store_sk#8] +Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight -(9) Project [codegen id : 5] -Output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +(7) CometProject Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] +Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(10) Scan parquet spark_catalog.default.customer_demographics +(8) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(9) CometFilter Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Condition : (isnotnull(cd_demo_sk#9) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = College )))) -(12) Filter [codegen id : 2] +(10) CometProject Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) AS cd_marital_status#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) AS cd_education_status#13] -(13) BroadcastExchange -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(11) CometBroadcastExchange +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))) +(12) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight -(15) Project [codegen id : 5] -Output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +(13) CometProject +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(16) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +(14) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [KY,MN,OR])),In(ca_state, [CA,MS,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] - -(18) Filter [codegen id : 3] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) +(15) CometFilter +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (VA,CA,MS))) -(19) Project [codegen id : 3] -Output [2]: [ca_address_sk#12, ca_state#13] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +(16) CometProject +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) AS ca_state#17] -(20) BroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#12] -Join type: Inner -Join condition: ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))) +(18) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight -(22) Project [codegen id : 5] -Output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] +(19) CometProject +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] +Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(23) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(20) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_year#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#15, d_year#16] +(21) CometFilter +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) -(25) Filter [codegen id : 4] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(22) CometProject +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#15] -Input [2]: [d_date_sk#15, d_year#16] +(23) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] -(27) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(24) CometBroadcastHashJoin +Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(25) CometProject +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] +Arguments: [ss_quantity#4], [ss_quantity#4] -(29) Project [codegen id : 5] -Output [1]: [ss_quantity#4] -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] - -(30) HashAggregate [codegen id : 5] +(26) CometHashAggregate Input [1]: [ss_quantity#4] Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum#17] -Results [1]: [sum#18] -(31) Exchange -Input [1]: [sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [1]: [sum#20] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [1]: [sum#18] +(28) CometHashAggregate +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum(ss_quantity#4)#19] -Results [1]: [sum(ss_quantity#4)#19 AS sum(ss_quantity)#20] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/extended.txt new file mode 100644 index 0000000000..bdbe1b496a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/simplified.txt index 2d1dbeadd4..5feebbec84 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/simplified.txt @@ -1,48 +1,31 @@ -WholeStageCodegen (6) - HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (5) - HashAggregate [ss_quantity] [sum,sum] - Project [ss_quantity] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sold_date_sk] - BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit] - Project [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price] - Project [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_cdemo_sk,ss_addr_sk,ss_sales_price,ss_net_profit] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum] [sum(ss_quantity),sum(ss_quantity)] + CometExchange #1 + CometHashAggregate [ss_quantity] [sum] + CometProject [ss_quantity] + CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] + CometProject [ss_quantity,ss_sold_date_sk] + CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/explain.txt index b024c6eed2..1d6b3ef43f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/explain.txt @@ -1,457 +1,425 @@ == Physical Plan == -TakeOrderedAndProject (81) -+- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- Union (77) - :- * Project (28) - : +- * Filter (27) - : +- Window (26) - : +- * Sort (25) - : +- Window (24) - : +- * Sort (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- Scan parquet spark_catalog.default.web_returns (6) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - :- * Project (52) - : +- * Filter (51) - : +- Window (50) - : +- * Sort (49) - : +- Window (48) - : +- * Sort (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildLeft (38) - : : :- BroadcastExchange (33) - : : : +- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.catalog_sales (29) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.catalog_returns (34) - : +- ReusedExchange (40) - +- * Project (76) - +- * Filter (75) - +- Window (74) - +- * Sort (73) - +- Window (72) - +- * Sort (71) - +- Exchange (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildLeft (62) - : :- BroadcastExchange (57) - : : +- * Project (56) - : : +- * Filter (55) - : : +- * ColumnarToRow (54) - : : +- Scan parquet spark_catalog.default.store_sales (53) - : +- * Project (61) - : +- * Filter (60) - : +- * ColumnarToRow (59) - : +- Scan parquet spark_catalog.default.store_returns (58) - +- ReusedExchange (64) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +(3) CometProject Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) BroadcastExchange +(4) CometBroadcastExchange Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(6) Scan parquet spark_catalog.default.web_returns +(5) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(7) ColumnarToRow -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(8) Filter +(6) CometFilter Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) -(9) Project -Output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +(7) CometProject Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#8, wr_item_sk#7] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +(9) CometProject Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(12) Scan parquet spark_catalog.default.date_dim +(10) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_moy#14] Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#12] +(12) CometProject Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12] -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#6] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight -(18) Project [codegen id : 3] -Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] +(15) CometProject Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -(19) HashAggregate [codegen id : 3] +(16) CometHashAggregate Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(20) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 4] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] -(22) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(23) Sort [codegen id : 5] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(24) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] -(25) Sort [codegen id : 6] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 +(22) Window +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] -(26) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 -(27) Filter [codegen id : 7] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) +(24) Window +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) -(28) Project [codegen id : 7] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(29) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(27) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(31) Filter [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : (((((((isnotnull(cs_net_profit#41) AND isnotnull(cs_net_paid#40)) AND isnotnull(cs_quantity#39)) AND (cs_net_profit#41 > 1.00)) AND (cs_net_paid#40 > 0.00)) AND (cs_quantity#39 > 0)) AND isnotnull(cs_order_number#38)) AND isnotnull(cs_item_sk#37)) +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) -(32) Project [codegen id : 8] -Output [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(33) BroadcastExchange -Input [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=5] +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(34) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] +(31) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(35) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(36) Filter -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) - -(37) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#38, cs_item_sk#37] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [6]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46] -Input [9]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] - -(40) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#48] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#42] -Right keys [1]: [d_date_sk#48] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Input [7]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] - -(43) HashAggregate [codegen id : 10] -Input [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Keys [1]: [cs_item_sk#37] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#39, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Results [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] - -(44) Exchange -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Keys [1]: [cs_item_sk#37] -Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#39, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#39, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64] -Results [3]: [cs_item_sk#37 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#39, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] - -(46) Exchange -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(47) Sort [codegen id : 12] -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 - -(48) Window -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] - -(49) Sort [codegen id : 13] -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 - -(50) Window -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] - -(51) Filter [codegen id : 14] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) - -(52) Project [codegen id : 14] -Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] - -(53) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(55) Filter [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) -(56) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(57) BroadcastExchange -Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=8] +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(58) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] +(54) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(59) ColumnarToRow -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(60) Filter -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Condition : (((isnotnull(sr_return_amt#80) AND (sr_return_amt#80 > 10000.00)) AND isnotnull(sr_ticket_number#78)) AND isnotnull(sr_item_sk#77)) - -(61) Project -Output [4]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(62) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#78, sr_item_sk#77] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80] -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] - -(64) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#82] - -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#76] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80, d_date_sk#82] - -(67) HashAggregate [codegen id : 17] -Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Keys [1]: [ss_item_sk#71] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#79, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#83, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] -Results [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] - -(68) Exchange -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(69) HashAggregate [codegen id : 18] -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Keys [1]: [ss_item_sk#71] -Functions [4]: [sum(coalesce(sr_return_quantity#79, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#79, 0))#95, sum(coalesce(ss_quantity#73, 0))#96, sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98] -Results [3]: [ss_item_sk#71 AS item#99, (cast(sum(coalesce(sr_return_quantity#79, 0))#95 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#96 as decimal(15,4))) AS return_ratio#100, (cast(sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98 as decimal(15,4))) AS currency_ratio#101] - -(70) Exchange -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(71) Sort [codegen id : 19] -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [return_ratio#100 ASC NULLS FIRST], false, 0 - -(72) Window -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [rank(return_ratio#100) windowspecdefinition(return_ratio#100 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#102], [return_ratio#100 ASC NULLS FIRST] - -(73) Sort [codegen id : 20] -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [currency_ratio#101 ASC NULLS FIRST], false, 0 - -(74) Window -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [rank(currency_ratio#101) windowspecdefinition(currency_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#103], [currency_ratio#101 ASC NULLS FIRST] - -(75) Filter [codegen id : 21] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] -Condition : ((return_rank#102 <= 10) OR (currency_rank#103 <= 10)) - -(76) Project [codegen id : 21] -Output [5]: [store AS channel#104, item#99, return_ratio#100, return_rank#102, currency_rank#103] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] - -(77) Union - -(78) HashAggregate [codegen id : 22] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(79) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(80) HashAggregate [codegen id : 23] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -(81) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/extended.txt new file mode 100644 index 0000000000..262571ed59 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/extended.txt @@ -0,0 +1,86 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- Filter + +- Window + +- Sort + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 63 out of 80 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/simplified.txt index 018748b274..117b74a75b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/simplified.txt @@ -1,129 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (7) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (6) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (5) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (4) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (14) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (13) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (12) - Sort [return_ratio] - InputAdapter - Exchange #6 - WholeStageCodegen (11) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #7 - WholeStageCodegen (10) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (20) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (19) - Sort [return_ratio] - InputAdapter - Exchange #9 - WholeStageCodegen (18) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (17) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/explain.txt index 31072e61eb..03a193564c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/explain.txt @@ -1,448 +1,409 @@ == Physical Plan == -TakeOrderedAndProject (78) -+- * HashAggregate (77) - +- Exchange (76) - +- * HashAggregate (75) - +- * Expand (74) - +- Union (73) - :- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- Union (9) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) - : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : +- BroadcastExchange (14) - : : +- * Project (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.date_dim (10) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - :- * HashAggregate (46) - : +- Exchange (45) - : +- * HashAggregate (44) - : +- * Project (43) - : +- * BroadcastHashJoin Inner BuildRight (42) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- Union (34) - : : : :- * Project (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (26) - : : : +- * Project (33) - : : : +- * Filter (32) - : : : +- * ColumnarToRow (31) - : : : +- Scan parquet spark_catalog.default.catalog_returns (30) - : : +- ReusedExchange (35) - : +- BroadcastExchange (41) - : +- * Filter (40) - : +- * ColumnarToRow (39) - : +- Scan parquet spark_catalog.default.catalog_page (38) - +- * HashAggregate (72) - +- Exchange (71) - +- * HashAggregate (70) - +- * Project (69) - +- * BroadcastHashJoin Inner BuildRight (68) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildRight (62) - : :- Union (60) - : : :- * Project (50) - : : : +- * Filter (49) - : : : +- * ColumnarToRow (48) - : : : +- Scan parquet spark_catalog.default.web_sales (47) - : : +- * Project (59) - : : +- * BroadcastHashJoin Inner BuildLeft (58) - : : :- BroadcastExchange (53) - : : : +- * ColumnarToRow (52) - : : : +- Scan parquet spark_catalog.default.web_returns (51) - : : +- * Project (57) - : : +- * Filter (56) - : : +- * ColumnarToRow (55) - : : +- Scan parquet spark_catalog.default.web_sales (54) - : +- ReusedExchange (61) - +- BroadcastExchange (67) - +- * Filter (66) - +- * ColumnarToRow (65) - +- Scan parquet spark_catalog.default.web_site (64) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometHashAggregate (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometExpand (66) + +- CometUnion (65) + :- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.store (14) + :- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometUnion (29) + : : : :- CometProject (25) + : : : : +- CometFilter (24) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (23) + : : : +- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (26) + : : +- ReusedExchange (30) + : +- CometBroadcastExchange (36) + : +- CometProject (35) + : +- CometFilter (34) + : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometUnion (52) + : : :- CometProject (44) + : : : +- CometFilter (43) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (42) + : : +- CometProject (51) + : : +- CometBroadcastHashJoin (50) + : : :- CometBroadcastExchange (46) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (45) + : : +- CometProject (49) + : : +- CometFilter (48) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (53) + +- CometBroadcastExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometNativeScan parquet spark_catalog.default.web_site (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] +(3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Condition : isnotnull(sr_store_sk#11) -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] +(6) CometProject Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] -(9) Union +(7) CometUnion +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(10) Scan parquet spark_catalog.default.date_dim +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] - -(12) Filter [codegen id : 3] +(9) CometFilter Input [2]: [d_date_sk#21, d_date#22] Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) -(13) Project [codegen id : 3] -Output [1]: [d_date_sk#21] +(10) CometProject Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#21] -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [date_sk#6] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight -(16) Project [codegen id : 5] -Output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +(13) CometProject Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(17) Scan parquet spark_catalog.default.store +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] - -(19) Filter [codegen id : 4] +(15) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(20) BroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] + +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [store_sk#5] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(22) Project [codegen id : 5] -Output [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(23) HashAggregate [codegen id : 5] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -(24) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 6] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] +(22) CometHashAggregate +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#39, store channel AS channel#40, concat(store, s_store_id#24) AS id#41] -(26) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(23) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(24) CometFilter +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : isnotnull(cs_catalog_page_sk#30) -(28) Filter [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : isnotnull(cs_catalog_page_sk#42) +(25) CometProject +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39], [cs_catalog_page_sk#30 AS page_sk#34, cs_sold_date_sk#33 AS date_sk#35, cs_ext_sales_price#31 AS sales_price#36, cs_net_profit#32 AS profit#37, 0.00 AS return_amt#38, 0.00 AS net_loss#39] -(29) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#46, cs_sold_date_sk#45 AS date_sk#47, cs_ext_sales_price#43 AS sales_price#48, cs_net_profit#44 AS profit#49, 0.00 AS return_amt#50, 0.00 AS net_loss#51] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(30) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(26) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#55)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] - -(32) Filter [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Condition : isnotnull(cr_catalog_page_sk#52) +(27) CometFilter +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Condition : isnotnull(cr_catalog_page_sk#40) -(33) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#52 AS page_sk#56, cr_returned_date_sk#55 AS date_sk#57, 0.00 AS sales_price#58, 0.00 AS profit#59, cr_return_amount#53 AS return_amt#60, cr_net_loss#54 AS net_loss#61] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(28) CometProject +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Arguments: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49], [cr_catalog_page_sk#40 AS page_sk#44, cr_returned_date_sk#43 AS date_sk#45, 0.00 AS sales_price#46, 0.00 AS profit#47, cr_return_amount#41 AS return_amt#48, cr_net_loss#42 AS net_loss#49] -(34) Union +(29) CometUnion +Child 0 Input [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Child 1 Input [6]: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49] -(35) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#62] +(30) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#50] -(36) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [date_sk#47] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None +(31) CometBroadcastHashJoin +Left output [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [1]: [d_date_sk#50] +Arguments: [date_sk#35], [d_date_sk#50], Inner, BuildRight -(37) Project [codegen id : 11] -Output [5]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51] -Input [7]: [page_sk#46, date_sk#47, sales_price#48, profit#49, return_amt#50, net_loss#51, d_date_sk#62] +(32) CometProject +Input [7]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39, d_date_sk#50] +Arguments: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39], [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] -(38) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] +(33) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(40) Filter [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Condition : isnotnull(cp_catalog_page_sk#63) - -(41) BroadcastExchange -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(42) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [page_sk#46] -Right keys [1]: [cp_catalog_page_sk#63] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 11] -Output [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Input [7]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(44) HashAggregate [codegen id : 11] -Input [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#48)), partial_sum(UnscaledValue(return_amt#50)), partial_sum(UnscaledValue(profit#49)), partial_sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum#65, sum#66, sum#67, sum#68] -Results [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] - -(45) Exchange -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(46) HashAggregate [codegen id : 12] -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#48)), sum(UnscaledValue(return_amt#50)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#48))#73, sum(UnscaledValue(return_amt#50))#74, sum(UnscaledValue(profit#49))#75, sum(UnscaledValue(net_loss#51))#76] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#48))#73,17,2) AS sales#77, MakeDecimal(sum(UnscaledValue(return_amt#50))#74,17,2) AS returns#78, (MakeDecimal(sum(UnscaledValue(profit#49))#75,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#51))#76,17,2)) AS profit#79, catalog channel AS channel#80, concat(catalog_page, cp_catalog_page_id#64) AS id#81] - -(47) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(34) CometFilter +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Condition : isnotnull(cp_catalog_page_sk#51) + +(35) CometProject +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53], [cp_catalog_page_sk#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#52, 16, true, false, true) AS cp_catalog_page_id#53] + +(36) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53] + +(37) CometBroadcastHashJoin +Left output [5]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [page_sk#34], [cp_catalog_page_sk#51], Inner, BuildRight + +(38) CometProject +Input [7]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53], [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] + +(39) CometHashAggregate +Input [5]: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [partial_sum(UnscaledValue(sales_price#36)), partial_sum(UnscaledValue(return_amt#38)), partial_sum(UnscaledValue(profit#37)), partial_sum(UnscaledValue(net_loss#39))] + +(40) CometExchange +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(41) CometHashAggregate +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [sum(UnscaledValue(sales_price#36)), sum(UnscaledValue(return_amt#38)), sum(UnscaledValue(profit#37)), sum(UnscaledValue(net_loss#39))] + +(42) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#85)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(49) Filter [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Condition : isnotnull(ws_web_site_sk#82) +(43) CometFilter +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_web_site_sk#58) -(50) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#82 AS wsr_web_site_sk#86, ws_sold_date_sk#85 AS date_sk#87, ws_ext_sales_price#83 AS sales_price#88, ws_net_profit#84 AS profit#89, 0.00 AS return_amt#90, 0.00 AS net_loss#91] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(44) CometProject +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Arguments: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67], [ws_web_site_sk#58 AS wsr_web_site_sk#62, ws_sold_date_sk#61 AS date_sk#63, ws_ext_sales_price#59 AS sales_price#64, ws_net_profit#60 AS profit#65, 0.00 AS return_amt#66, 0.00 AS net_loss#67] -(51) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] +(45) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#96)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#72)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] +(46) CometBroadcastExchange +Input [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Arguments: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] -(53) BroadcastExchange -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=6] - -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(55) ColumnarToRow -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] - -(56) Filter -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Condition : ((isnotnull(ws_item_sk#97) AND isnotnull(ws_order_number#99)) AND isnotnull(ws_web_site_sk#98)) +(48) CometFilter +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Condition : ((isnotnull(ws_item_sk#73) AND isnotnull(ws_order_number#75)) AND isnotnull(ws_web_site_sk#74)) -(57) Project -Output [3]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(49) CometProject +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Arguments: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75], [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] -(58) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#92, wr_order_number#93] -Right keys [2]: [ws_item_sk#97, ws_order_number#99] -Join type: Inner -Join condition: None +(50) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Right output [3]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wr_item_sk#68, wr_order_number#69], [ws_item_sk#73, ws_order_number#75], Inner, BuildLeft -(59) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#98 AS wsr_web_site_sk#101, wr_returned_date_sk#96 AS date_sk#102, 0.00 AS sales_price#103, 0.00 AS profit#104, wr_return_amt#94 AS return_amt#105, wr_net_loss#95 AS net_loss#106] -Input [8]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96, ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] +(51) CometProject +Input [8]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72, ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82], [ws_web_site_sk#74 AS wsr_web_site_sk#77, wr_returned_date_sk#72 AS date_sk#78, 0.00 AS sales_price#79, 0.00 AS profit#80, wr_return_amt#70 AS return_amt#81, wr_net_loss#71 AS net_loss#82] -(60) Union +(52) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Child 1 Input [6]: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82] -(61) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#107] +(53) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#83] -(62) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [date_sk#87] -Right keys [1]: [d_date_sk#107] -Join type: Inner -Join condition: None +(54) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [1]: [d_date_sk#83] +Arguments: [date_sk#63], [d_date_sk#83], Inner, BuildRight -(63) Project [codegen id : 18] -Output [5]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91] -Input [7]: [wsr_web_site_sk#86, date_sk#87, sales_price#88, profit#89, return_amt#90, net_loss#91, d_date_sk#107] +(55) CometProject +Input [7]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67, d_date_sk#83] +Arguments: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67], [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] -(64) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#108, web_site_id#109] +(56) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#84, web_site_id#85] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] - -(66) Filter [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] -Condition : isnotnull(web_site_sk#108) - -(67) BroadcastExchange -Input [2]: [web_site_sk#108, web_site_id#109] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(68) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [wsr_web_site_sk#86] -Right keys [1]: [web_site_sk#108] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 18] -Output [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Input [7]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_sk#108, web_site_id#109] - -(70) HashAggregate [codegen id : 18] -Input [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Keys [1]: [web_site_id#109] -Functions [4]: [partial_sum(UnscaledValue(sales_price#88)), partial_sum(UnscaledValue(return_amt#90)), partial_sum(UnscaledValue(profit#89)), partial_sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum#110, sum#111, sum#112, sum#113] -Results [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] - -(71) Exchange -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Arguments: hashpartitioning(web_site_id#109, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(72) HashAggregate [codegen id : 19] -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Keys [1]: [web_site_id#109] -Functions [4]: [sum(UnscaledValue(sales_price#88)), sum(UnscaledValue(return_amt#90)), sum(UnscaledValue(profit#89)), sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#88))#118, sum(UnscaledValue(return_amt#90))#119, sum(UnscaledValue(profit#89))#120, sum(UnscaledValue(net_loss#91))#121] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#88))#118,17,2) AS sales#122, MakeDecimal(sum(UnscaledValue(return_amt#90))#119,17,2) AS returns#123, (MakeDecimal(sum(UnscaledValue(profit#89))#120,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#91))#121,17,2)) AS profit#124, web channel AS channel#125, concat(web_site, web_site_id#109) AS id#126] - -(73) Union - -(74) Expand [codegen id : 20] -Input [5]: [sales#37, returns#38, profit#39, channel#40, id#41] -Arguments: [[sales#37, returns#38, profit#39, channel#40, id#41, 0], [sales#37, returns#38, profit#39, channel#40, null, 1], [sales#37, returns#38, profit#39, null, null, 3]], [sales#37, returns#38, profit#39, channel#127, id#128, spark_grouping_id#129] - -(75) HashAggregate [codegen id : 20] -Input [6]: [sales#37, returns#38, profit#39, channel#127, id#128, spark_grouping_id#129] -Keys [3]: [channel#127, id#128, spark_grouping_id#129] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Results [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] - -(76) Exchange -Input [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Arguments: hashpartitioning(channel#127, id#128, spark_grouping_id#129, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(77) HashAggregate [codegen id : 21] -Input [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Keys [3]: [channel#127, id#128, spark_grouping_id#129] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#142, sum(returns#38)#143, sum(profit#39)#144] -Results [5]: [channel#127, id#128, sum(sales#37)#142 AS sales#145, sum(returns#38)#143 AS returns#146, sum(profit#39)#144 AS profit#147] - -(78) TakeOrderedAndProject -Input [5]: [channel#127, id#128, sales#145, returns#146, profit#147] -Arguments: 100, [channel#127 ASC NULLS FIRST, id#128 ASC NULLS FIRST], [channel#127, id#128, sales#145, returns#146, profit#147] +(57) CometFilter +Input [2]: [web_site_sk#84, web_site_id#85] +Condition : isnotnull(web_site_sk#84) + +(58) CometProject +Input [2]: [web_site_sk#84, web_site_id#85] +Arguments: [web_site_sk#84, web_site_id#86], [web_site_sk#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#85, 16, true, false, true) AS web_site_id#86] + +(59) CometBroadcastExchange +Input [2]: [web_site_sk#84, web_site_id#86] +Arguments: [web_site_sk#84, web_site_id#86] + +(60) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [2]: [web_site_sk#84, web_site_id#86] +Arguments: [wsr_web_site_sk#62], [web_site_sk#84], Inner, BuildRight + +(61) CometProject +Input [7]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_sk#84, web_site_id#86] +Arguments: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86], [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] + +(62) CometHashAggregate +Input [5]: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] +Keys [1]: [web_site_id#86] +Functions [4]: [partial_sum(UnscaledValue(sales_price#64)), partial_sum(UnscaledValue(return_amt#66)), partial_sum(UnscaledValue(profit#65)), partial_sum(UnscaledValue(net_loss#67))] + +(63) CometExchange +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Arguments: hashpartitioning(web_site_id#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(64) CometHashAggregate +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Keys [1]: [web_site_id#86] +Functions [4]: [sum(UnscaledValue(sales_price#64)), sum(UnscaledValue(return_amt#66)), sum(UnscaledValue(profit#65)), sum(UnscaledValue(net_loss#67))] + +(65) CometUnion +Child 0 Input [5]: [sales#91, returns#92, profit#93, channel#94, id#95] +Child 1 Input [5]: [sales#96, returns#97, profit#98, channel#99, id#100] +Child 2 Input [5]: [sales#101, returns#102, profit#103, channel#104, id#105] + +(66) CometExpand +Input [5]: [sales#91, returns#92, profit#93, channel#94, id#95] +Arguments: [[sales#91, returns#92, profit#93, channel#94, id#95, 0], [sales#91, returns#92, profit#93, channel#94, null, 1], [sales#91, returns#92, profit#93, null, null, 3]], [sales#91, returns#92, profit#93, channel#106, id#107, spark_grouping_id#108] + +(67) CometHashAggregate +Input [6]: [sales#91, returns#92, profit#93, channel#106, id#107, spark_grouping_id#108] +Keys [3]: [channel#106, id#107, spark_grouping_id#108] +Functions [3]: [partial_sum(sales#91), partial_sum(returns#92), partial_sum(profit#93)] + +(68) CometExchange +Input [9]: [channel#106, id#107, spark_grouping_id#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Arguments: hashpartitioning(channel#106, id#107, spark_grouping_id#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(69) CometHashAggregate +Input [9]: [channel#106, id#107, spark_grouping_id#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Keys [3]: [channel#106, id#107, spark_grouping_id#108] +Functions [3]: [sum(sales#91), sum(returns#92), sum(profit#93)] + +(70) CometTakeOrderedAndProject +Input [5]: [channel#106, id#107, sales#115, returns#116, profit#117] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#106 ASC NULLS FIRST,id#107 ASC NULLS FIRST], output=[channel#106,id#107,sales#115,returns#116,profit#117]), [channel#106, id#107, sales#115, returns#116, profit#117], 100, 0, [channel#106 ASC NULLS FIRST, id#107 ASC NULLS FIRST], [channel#106, id#107, sales#115, returns#116, profit#117] + +(71) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#106, id#107, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/extended.txt new file mode 100644 index 0000000000..ebcc9b408b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/extended.txt @@ -0,0 +1,79 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_page + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 76 out of 76 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/simplified.txt index 9893a05dca..09633b6ce5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/simplified.txt @@ -1,123 +1,73 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (21) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (20) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (5) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,s_store_id] - BroadcastHashJoin [store_sk,s_store_sk] - Project [store_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [cp_catalog_page_id] #5 - WholeStageCodegen (11) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - BroadcastHashJoin [page_sk,cp_catalog_page_sk] - Project [page_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #7 - WholeStageCodegen (18) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,web_site_id] - BroadcastHashJoin [wsr_web_site_sk,web_site_sk] - Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id,spark_grouping_id] #1 + CometHashAggregate [sales,returns,profit] [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometExpand [channel,id] [sales,returns,profit,channel,id,spark_grouping_id] + CometUnion [sales,returns,profit,channel,id] + CometHashAggregate [sum,sum,sum,sum] [sales,returns,profit,channel,id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),s_store_id] + CometExchange [s_store_id] #2 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometHashAggregate [sum,sum,sum,sum] [sales,returns,profit,channel,id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),cp_catalog_page_id] + CometExchange [cp_catalog_page_id] #5 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometHashAggregate [sum,sum,sum,sum] [sales,returns,profit,channel,id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),web_site_id] + CometExchange [web_site_id] #7 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/explain.txt index 7214b3b58e..1fe7a0246b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/explain.txt @@ -1,189 +1,170 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.store (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#9)] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(input[2, int, false], input[0, int, false], input[1, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [3]: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7], Inner, BuildRight -(9) Project [codegen id : 5] -Output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] +(7) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -(10) Scan parquet spark_catalog.default.store +(8) CometNativeScan parquet spark_catalog.default.store Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Condition : isnotnull(s_store_sk#10) -(13) BroadcastExchange +(10) CometProject Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_street_type#15, 15, true, false, true) AS s_street_type#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_suite_number#16, 10, true, false, true) AS s_suite_number#22, s_city#17, s_county#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#19, 2, true, false, true) AS s_state#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#20, 10, true, false, true) AS s_zip#24] + +(11) CometBroadcastExchange +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(15) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(13) CometProject +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(16) Scan parquet spark_catalog.default.date_dim -Output [1]: [d_date_sk#21] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [1]: [d_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [1]: [d_date_sk#21] - -(18) Filter [codegen id : 3] -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) +(15) CometFilter +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) -(19) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight -(21) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] +(18) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(22) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] +(20) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) -(24) Filter [codegen id : 4] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) +(21) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#22] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] -(26) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_returned_date_sk#9] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None +(24) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(28) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] - -(29) HashAggregate [codegen id : 5] -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(25) CometHashAggregate +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#25, sum#26, sum#27, sum#28, sum#29] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -(30) Exchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 6] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(27) CometHashAggregate +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35 AS 30 days #40, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36 AS 31 - 60 days #41, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37 AS 61 - 90 days #42, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38 AS 91 - 120 days #43, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39 AS >120 days #44] -(32) TakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] -Arguments: 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] +(28) CometTakeOrderedAndProject +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] + +(29) CometColumnarToRow [codegen id : 1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/extended.txt new file mode 100644 index 0000000000..a188fea58c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/simplified.txt index 0108f64701..e53cdee055 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/simplified.txt @@ -1,48 +1,31 @@ -TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - WholeStageCodegen (5) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,ss_customer_sk,sr_ticket_number,sr_item_sk,sr_customer_sk] - Filter [ss_ticket_number,ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_ticket_number,sr_item_sk,sr_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 + CometHashAggregate [sr_returned_date_sk,ss_sold_date_sk] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 + CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastExchange [d_date_sk] #4 + CometFilter [d_date_sk] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/explain.txt index 3bb6c1d637..f9c8d14c62 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/explain.txt @@ -2,232 +2,221 @@ TakeOrderedAndProject (41) +- * Filter (40) +- Window (39) - +- * Sort (38) - +- Exchange (37) - +- * Project (36) - +- * SortMergeJoin FullOuter (35) - :- * Sort (19) - : +- Exchange (18) - : +- * Project (17) - : +- Window (16) - : +- * Sort (15) - : +- Exchange (14) - : +- * HashAggregate (13) - : +- Exchange (12) - : +- * HashAggregate (11) - : +- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- * Sort (34) - +- Exchange (33) - +- * Project (32) - +- Window (31) - +- * Sort (30) - +- Exchange (29) - +- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Filter (22) - : +- * ColumnarToRow (21) - : +- Scan parquet spark_catalog.default.store_sales (20) - +- ReusedExchange (23) - - -(1) Scan parquet spark_catalog.default.web_sales + +- * CometColumnarToRow (38) + +- CometSort (37) + +- CometExchange (36) + +- CometProject (35) + +- CometSortMergeJoin (34) + :- CometSort (18) + : +- CometColumnarExchange (17) + : +- * Project (16) + : +- Window (15) + : +- * CometColumnarToRow (14) + : +- CometSort (13) + : +- CometExchange (12) + : +- CometHashAggregate (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- CometSort (33) + +- CometColumnarExchange (32) + +- * Project (31) + +- Window (30) + +- * CometColumnarToRow (29) + +- CometSort (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometFilter (20) + : +- CometNativeScan parquet spark_catalog.default.store_sales (19) + +- ReusedExchange (21) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] +(5) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_date#5] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 2] -Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +(8) CometProject Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] -(11) HashAggregate [codegen id : 2] +(9) CometHashAggregate Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] -(12) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) HashAggregate [codegen id : 3] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] +(11) CometHashAggregate +Input [3]: [ws_item_sk#1, d_date#5, sum#7] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS _w0#11, ws_item_sk#1] -(14) Exchange -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(12) CometExchange +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(13) CometSort +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(15) Sort [codegen id : 4] -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +(14) CometColumnarToRow [codegen id : 1] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] -(16) Window -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [sum(_w0#11) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] +(15) Window +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [sum(_w0#9) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] -(17) Project [codegen id : 5] -Output [3]: [item_sk#10, d_date#5, cume_sales#12] -Input [5]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1, cume_sales#12] +(16) Project [codegen id : 2] +Output [3]: [item_sk#8, d_date#5, cume_sales#10] +Input [5]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1, cume_sales#10] -(18) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(17) CometColumnarExchange +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(19) Sort [codegen id : 6] -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +(18) CometSort +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: [item_sk#8, d_date#5, cume_sales#10], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(20) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] +(19) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] - -(22) Filter [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_item_sk#13) - -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#16, d_date#17] - -(24) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 8] -Output [3]: [ss_item_sk#13, ss_sales_price#14, d_date#17] -Input [5]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_date#17] - -(26) HashAggregate [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, d_date#17] -Keys [2]: [ss_item_sk#13, d_date#17] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#14))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ss_item_sk#13, d_date#17, sum#19] - -(27) Exchange -Input [3]: [ss_item_sk#13, d_date#17, sum#19] -Arguments: hashpartitioning(ss_item_sk#13, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(28) HashAggregate [codegen id : 9] -Input [3]: [ss_item_sk#13, d_date#17, sum#19] -Keys [2]: [ss_item_sk#13, d_date#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#14))#20] -Results [4]: [ss_item_sk#13 AS item_sk#21, d_date#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#14))#20,17,2) AS _w0#22, ss_item_sk#13] - -(29) Exchange -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: hashpartitioning(ss_item_sk#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(30) Sort [codegen id : 10] -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: [ss_item_sk#13 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(31) Window -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: [sum(_w0#22) windowspecdefinition(ss_item_sk#13, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#23], [ss_item_sk#13], [d_date#17 ASC NULLS FIRST] - -(32) Project [codegen id : 11] -Output [3]: [item_sk#21, d_date#17, cume_sales#23] -Input [5]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13, cume_sales#23] - -(33) Exchange -Input [3]: [item_sk#21, d_date#17, cume_sales#23] -Arguments: hashpartitioning(item_sk#21, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(34) Sort [codegen id : 12] -Input [3]: [item_sk#21, d_date#17, cume_sales#23] -Arguments: [item_sk#21 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(35) SortMergeJoin [codegen id : 13] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#21, d_date#17] -Join type: FullOuter -Join condition: None - -(36) Project [codegen id : 13] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#21 END AS item_sk#24, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#17 END AS d_date#25, cume_sales#12 AS web_sales#26, cume_sales#23 AS store_sales#27] -Input [6]: [item_sk#10, d_date#5, cume_sales#12, item_sk#21, d_date#17, cume_sales#23] - -(37) Exchange -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: hashpartitioning(item_sk#24, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(38) Sort [codegen id : 14] -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST], false, 0 +(20) CometFilter +Input [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#11) + +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#14, d_date#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_date#15] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(23) CometProject +Input [5]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_date#15] +Arguments: [ss_item_sk#11, ss_sales_price#12, d_date#15], [ss_item_sk#11, ss_sales_price#12, d_date#15] + +(24) CometHashAggregate +Input [3]: [ss_item_sk#11, ss_sales_price#12, d_date#15] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] + +(25) CometExchange +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Arguments: hashpartitioning(ss_item_sk#11, d_date#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometHashAggregate +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] + +(27) CometExchange +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: hashpartitioning(ss_item_sk#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(28) CometSort +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11], [ss_item_sk#11 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] + +(29) CometColumnarToRow [codegen id : 3] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] + +(30) Window +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [sum(_w0#18) windowspecdefinition(ss_item_sk#11, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#19], [ss_item_sk#11], [d_date#15 ASC NULLS FIRST] + +(31) Project [codegen id : 4] +Output [3]: [item_sk#17, d_date#15, cume_sales#19] +Input [5]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11, cume_sales#19] + +(32) CometColumnarExchange +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: hashpartitioning(item_sk#17, d_date#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(33) CometSort +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#17, d_date#15, cume_sales#19], [item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] + +(34) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#10] +Right output [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#8, d_date#5], [item_sk#17, d_date#15], FullOuter + +(35) CometProject +Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#17 END AS item_sk#20, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#15 END AS d_date#21, cume_sales#10 AS web_sales#22, cume_sales#19 AS store_sales#23] + +(36) CometExchange +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: hashpartitioning(item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(37) CometSort +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST] + +(38) CometColumnarToRow [codegen id : 5] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] (39) Window -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [max(web_sales#26) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#28, max(store_sales#27) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#29], [item_sk#24], [d_date#25 ASC NULLS FIRST] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [max(web_sales#22) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#24, max(store_sales#23) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#25], [item_sk#20], [d_date#21 ASC NULLS FIRST] -(40) Filter [codegen id : 15] -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Condition : ((isnotnull(web_cumulative#28) AND isnotnull(store_cumulative#29)) AND (web_cumulative#28 > store_cumulative#29)) +(40) Filter [codegen id : 6] +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Condition : ((isnotnull(web_cumulative#24) AND isnotnull(store_cumulative#25)) AND (web_cumulative#24 > store_cumulative#25)) (41) TakeOrderedAndProject -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Arguments: 100, [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST], [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Arguments: 100, [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST], [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/extended.txt new file mode 100644 index 0000000000..08195fdffe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/extended.txt @@ -0,0 +1,46 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometColumnarExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 34 out of 41 eligible operators (82%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/simplified.txt index dd597f4e7b..300f5044c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/simplified.txt @@ -1,72 +1,53 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (15) + WholeStageCodegen (6) Filter [web_cumulative,store_cumulative] InputAdapter Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (14) - Sort [item_sk,d_date] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (13) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (5) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (4) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #4 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - Project [ws_item_sk,ws_sales_price,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #6 - WholeStageCodegen (11) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ss_item_sk,d_date] - WholeStageCodegen (10) - Sort [ss_item_sk,d_date] - InputAdapter - Exchange [ss_item_sk] #7 - WholeStageCodegen (9) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ss_item_sk,d_date] #8 - WholeStageCodegen (8) - HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_sales_price,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #5 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (2) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometExchange [ws_item_sk] #3 + CometHashAggregate [sum] [item_sk,d_date,_w0,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #6 + WholeStageCodegen (4) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometExchange [ss_item_sk] #7 + CometHashAggregate [sum] [item_sk,d_date,_w0,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #8 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/explain.txt index 2cb91028d9..dec8071d59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/explain.txt @@ -1,125 +1,111 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.item (9) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(19) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(17) CometHashAggregate +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] -(21) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, ext_price#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, ext_price#16] +(18) CometTakeOrderedAndProject +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,ext_price#15]), [d_year#2, brand_id#13, brand#14, ext_price#15], 100, 0, [d_year#2 ASC NULLS FIRST, ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, ext_price#15] + +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/simplified.txt index 16e313682b..cee223d027 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [d_year,ext_price,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [d_year,brand_id,brand,ext_price] + CometHashAggregate [sum] [d_year,brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/explain.txt index dea00fce8d..e09d5b7575 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/explain.txt @@ -1,184 +1,165 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (17) - : +- * BroadcastHashJoin Inner BuildRight (16) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.store_sales (5) - : +- BroadcastExchange (15) - : +- * Project (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.date_dim (11) - +- BroadcastExchange (21) - +- * Filter (20) - +- * ColumnarToRow (19) - +- Scan parquet spark_catalog.default.store (18) - - -(1) Scan parquet spark_catalog.default.item +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,reference ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(3) Filter [codegen id : 4] +(3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manufact_id#5] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] - -(5) Scan parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [i_item_sk#1, i_manufact_id#5] +Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +(8) CometProject Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) -(14) Project [codegen id : 2] -Output [2]: [d_date_sk#14, d_qoy#16] +(11) CometProject Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [d_date_sk#14, d_qoy#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#14, d_qoy#16] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_qoy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(17) Project [codegen id : 4] -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +(14) CometProject Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -(18) Scan parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [s_store_sk#17] Condition : isnotnull(s_store_sk#17) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [s_store_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#17] -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight -(23) Project [codegen id : 4] -Output [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] +(19) CometProject Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] -(24) HashAggregate [codegen id : 4] +(20) CometHashAggregate Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#19] -(25) Exchange -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] +(22) CometHashAggregate +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] -(27) Exchange -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometExchange +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [i_manufact_id#5, sum_sales#19, _w0#20], [i_manufact_id#5 ASC NULLS FIRST] -(28) Sort [codegen id : 6] -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] -(29) Window -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#23], [i_manufact_id#5] +(26) Window +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#21], [i_manufact_id#5] -(30) Filter [codegen id : 7] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] -Condition : CASE WHEN (avg_quarterly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_quarterly_sales#23)) / avg_quarterly_sales#23) > 0.1000000000000000) ELSE false END +(27) Filter [codegen id : 2] +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] +Condition : CASE WHEN (avg_quarterly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_quarterly_sales#21)) / avg_quarterly_sales#21) > 0.1000000000000000) ELSE false END -(31) Project [codegen id : 7] -Output [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] +(28) Project [codegen id : 2] +Output [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] -(32) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Arguments: 100, [avg_quarterly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] +(29) TakeOrderedAndProject +Input [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Arguments: 100, [avg_quarterly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/extended.txt new file mode 100644 index 0000000000..44fd48691b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/extended.txt @@ -0,0 +1,31 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/simplified.txt index 3fa8af3ff4..7e4a2be5ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/simplified.txt @@ -1,49 +1,33 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [i_manufact_id,sum_sales,avg_quarterly_sales] Filter [avg_quarterly_sales,sum_sales] InputAdapter Window [_w0,i_manufact_id] - WholeStageCodegen (6) - Sort [i_manufact_id] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (5) - HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manufact_id,d_qoy] #2 - WholeStageCodegen (4) - HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum] - Project [i_manufact_id,ss_sales_price,d_qoy] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manufact_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] + CometSort [i_manufact_id,sum_sales,_w0] + CometExchange [i_manufact_id] #1 + CometHashAggregate [d_qoy,sum] [i_manufact_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_manufact_id,d_qoy] #2 + CometHashAggregate [ss_sales_price] [i_manufact_id,d_qoy,sum] + CometProject [i_manufact_id,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manufact_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_qoy] #4 + CometProject [d_date_sk,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/explain.txt index 7d04d75608..703f2d4000 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/explain.txt @@ -1,469 +1,422 @@ == Physical Plan == -TakeOrderedAndProject (64) -+- * HashAggregate (63) - +- Exchange (62) - +- * HashAggregate (61) - +- * HashAggregate (60) - +- Exchange (59) - +- * HashAggregate (58) - +- * Project (57) - +- * BroadcastHashJoin Inner BuildRight (56) - :- * Project (50) - : +- * BroadcastHashJoin Inner BuildRight (49) - : :- * Project (44) - : : +- * BroadcastHashJoin Inner BuildRight (43) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * HashAggregate (32) - : : : : +- Exchange (31) - : : : : +- * HashAggregate (30) - : : : : +- * Project (29) - : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : :- * Project (23) - : : : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : : : :- * Project (16) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : : :- Union (9) - : : : : : : : :- * Project (4) - : : : : : : : : +- * Filter (3) - : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : +- * Project (8) - : : : : : : : +- * Filter (7) - : : : : : : : +- * ColumnarToRow (6) - : : : : : : : +- Scan parquet spark_catalog.default.web_sales (5) - : : : : : : +- BroadcastExchange (14) - : : : : : : +- * Project (13) - : : : : : : +- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (21) - : : : : : +- * Project (20) - : : : : : +- * Filter (19) - : : : : : +- * ColumnarToRow (18) - : : : : : +- Scan parquet spark_catalog.default.date_dim (17) - : : : : +- BroadcastExchange (27) - : : : : +- * Filter (26) - : : : : +- * ColumnarToRow (25) - : : : : +- Scan parquet spark_catalog.default.customer (24) - : : : +- BroadcastExchange (36) - : : : +- * Filter (35) - : : : +- * ColumnarToRow (34) - : : : +- Scan parquet spark_catalog.default.store_sales (33) - : : +- BroadcastExchange (42) - : : +- * Filter (41) - : : +- * ColumnarToRow (40) - : : +- Scan parquet spark_catalog.default.customer_address (39) - : +- BroadcastExchange (48) - : +- * Filter (47) - : +- * ColumnarToRow (46) - : +- Scan parquet spark_catalog.default.store (45) - +- BroadcastExchange (55) - +- * Project (54) - +- * Filter (53) - +- * ColumnarToRow (52) - +- Scan parquet spark_catalog.default.date_dim (51) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (58) ++- CometTakeOrderedAndProject (57) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometHashAggregate (53) + +- CometExchange (52) + +- CometHashAggregate (51) + +- CometProject (50) + +- CometBroadcastHashJoin (49) + :- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (38) + : : +- CometBroadcastHashJoin (37) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometHashAggregate (27) + : : : : +- CometExchange (26) + : : : : +- CometHashAggregate (25) + : : : : +- CometProject (24) + : : : : +- CometBroadcastHashJoin (23) + : : : : :- CometProject (19) + : : : : : +- CometBroadcastHashJoin (18) + : : : : : :- CometProject (13) + : : : : : : +- CometBroadcastHashJoin (12) + : : : : : : :- CometUnion (7) + : : : : : : : :- CometProject (3) + : : : : : : : : +- CometFilter (2) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometProject (6) + : : : : : : : +- CometFilter (5) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (4) + : : : : : : +- CometBroadcastExchange (11) + : : : : : : +- CometProject (10) + : : : : : : +- CometFilter (9) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (8) + : : : : : +- CometBroadcastExchange (17) + : : : : : +- CometProject (16) + : : : : : +- CometFilter (15) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + : : : : +- CometBroadcastExchange (22) + : : : : +- CometFilter (21) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (20) + : : : +- CometBroadcastExchange (30) + : : : +- CometFilter (29) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (28) + : : +- CometBroadcastExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (33) + : +- CometBroadcastExchange (42) + : +- CometProject (41) + : +- CometFilter (40) + : +- CometNativeScan parquet spark_catalog.default.store (39) + +- CometBroadcastExchange (48) + +- CometProject (47) + +- CometFilter (46) + +- CometNativeScan parquet spark_catalog.default.date_dim (45) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) -(4) Project [codegen id : 1] -Output [3]: [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] +(3) CometProject Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] +Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] -(5) Scan parquet spark_catalog.default.web_sales +(4) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) -(8) Project [codegen id : 2] -Output [3]: [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] +(6) CometProject Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] -(9) Union +(7) CometUnion +Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] -(10) Scan parquet spark_catalog.default.item +(8) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#13, i_class#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women ), EqualTo(i_class,maternity ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [3]: [i_item_sk#13, i_class#14, i_category#15] - -(12) Filter [codegen id : 3] +(9) CometFilter Input [3]: [i_item_sk#13, i_class#14, i_category#15] -Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (i_category#15 = Women )) AND (i_class#14 = maternity )) AND isnotnull(i_item_sk#13)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#14, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#13)) -(13) Project [codegen id : 3] -Output [1]: [i_item_sk#13] +(10) CometProject Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Arguments: [i_item_sk#13], [i_item_sk#13] -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [i_item_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#13] -(15) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_sk#6] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight -(16) Project [codegen id : 6] -Output [2]: [sold_date_sk#4, customer_sk#5] +(13) CometProject Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] +Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] -(17) Scan parquet spark_catalog.default.date_dim +(14) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(19) Filter [codegen id : 4] +(15) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) -(20) Project [codegen id : 4] -Output [1]: [d_date_sk#16] +(16) CometProject Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#16] -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sold_date_sk#4] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [2]: [sold_date_sk#4, customer_sk#5] +Right output [1]: [d_date_sk#16] +Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight -(23) Project [codegen id : 6] -Output [1]: [customer_sk#5] +(19) CometProject Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] +Arguments: [customer_sk#5], [customer_sk#5] -(24) Scan parquet spark_catalog.default.customer +(20) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#19, c_current_addr_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 5] -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] - -(26) Filter [codegen id : 5] +(21) CometFilter Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) -(27) BroadcastExchange +(22) CometBroadcastExchange Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [c_customer_sk#19, c_current_addr_sk#20] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [customer_sk#5] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [1]: [customer_sk#5] +Right output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [customer_sk#5], [c_customer_sk#19], Inner, BuildRight -(29) Project [codegen id : 6] -Output [2]: [c_customer_sk#19, c_current_addr_sk#20] +(24) CometProject Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20], [c_customer_sk#19, c_current_addr_sk#20] -(30) HashAggregate [codegen id : 6] +(25) CometHashAggregate Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] -Aggregate Attributes: [] -Results [2]: [c_customer_sk#19, c_current_addr_sk#20] -(31) Exchange +(26) CometExchange Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 11] +(27) CometHashAggregate Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] -Aggregate Attributes: [] -Results [2]: [c_customer_sk#19, c_current_addr_sk#20] -(33) Scan parquet spark_catalog.default.store_sales +(28) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 7] -Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(35) Filter [codegen id : 7] +(29) CometFilter Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Condition : isnotnull(ss_customer_sk#21) -(36) BroadcastExchange +(30) CometBroadcastExchange Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_customer_sk#19] -Right keys [1]: [ss_customer_sk#21] -Join type: Inner -Join condition: None +(31) CometBroadcastHashJoin +Left output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Right output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight -(38) Project [codegen id : 11] -Output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +(32) CometProject Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -(39) Scan parquet spark_catalog.default.customer_address +(33) CometNativeScan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county), IsNotNull(ca_state)] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 8] +(34) CometFilter Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true))) -(41) Filter [codegen id : 8] +(35) CometProject Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true) AS ca_state#27] -(42) BroadcastExchange -Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(36) CometBroadcastExchange +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#20] -Right keys [1]: [ca_address_sk#24] -Join type: Inner -Join condition: None +(37) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight -(44) Project [codegen id : 11] -Output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] -Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#26] +(38) CometProject +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] -(45) Scan parquet spark_catalog.default.store -Output [2]: [s_county#27, s_state#28] +(39) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_county#28, s_state#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_county), IsNotNull(s_state)] +PushedFilters: [IsNotNull(s_county)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 9] -Input [2]: [s_county#27, s_state#28] +(40) CometFilter +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true))) -(47) Filter [codegen id : 9] -Input [2]: [s_county#27, s_state#28] -Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) +(41) CometProject +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true) AS s_state#30] -(48) BroadcastExchange -Input [2]: [s_county#27, s_state#28] -Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=7] +(42) CometBroadcastExchange +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] -(49) BroadcastHashJoin [codegen id : 11] -Left keys [2]: [ca_county#25, ca_state#26] -Right keys [2]: [s_county#27, s_state#28] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight -(50) Project [codegen id : 11] -Output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26, s_county#27, s_state#28] +(44) CometProject +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -(51) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_month_seq#30] +(45) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_month_seq#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#29, d_month_seq#30] - -(53) Filter [codegen id : 10] -Input [2]: [d_date_sk#29, d_month_seq#30] -Condition : (((isnotnull(d_month_seq#30) AND (d_month_seq#30 >= Subquery scalar-subquery#31, [id=#32])) AND (d_month_seq#30 <= Subquery scalar-subquery#33, [id=#34])) AND isnotnull(d_date_sk#29)) +(46) CometFilter +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= Subquery scalar-subquery#33, [id=#34])) AND (d_month_seq#32 <= Subquery scalar-subquery#35, [id=#36])) AND isnotnull(d_date_sk#31)) -(54) Project [codegen id : 10] -Output [1]: [d_date_sk#29] -Input [2]: [d_date_sk#29, d_month_seq#30] +(47) CometProject +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] -(55) BroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(48) CometBroadcastExchange +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] -(56) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_sold_date_sk#23] -Right keys [1]: [d_date_sk#29] -Join type: Inner -Join condition: None +(49) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight -(57) Project [codegen id : 11] -Output [2]: [c_customer_sk#19, ss_ext_sales_price#22] -Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#29] +(50) CometProject +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] -(58) HashAggregate [codegen id : 11] +(51) CometHashAggregate Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] Keys [1]: [c_customer_sk#19] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] -Aggregate Attributes [1]: [sum#35] -Results [2]: [c_customer_sk#19, sum#36] -(59) Exchange -Input [2]: [c_customer_sk#19, sum#36] -Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(52) CometExchange +Input [2]: [c_customer_sk#19, sum#37] +Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(60) HashAggregate [codegen id : 12] -Input [2]: [c_customer_sk#19, sum#36] +(53) CometHashAggregate +Input [2]: [c_customer_sk#19, sum#37] Keys [1]: [c_customer_sk#19] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#22))#37] -Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#22))#37,17,2) / 50) as int) AS segment#38] -(61) HashAggregate [codegen id : 12] +(54) CometHashAggregate Input [1]: [segment#38] Keys [1]: [segment#38] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#39] -Results [2]: [segment#38, count#40] -(62) Exchange -Input [2]: [segment#38, count#40] -Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(55) CometExchange +Input [2]: [segment#38, count#39] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(63) HashAggregate [codegen id : 13] -Input [2]: [segment#38, count#40] +(56) CometHashAggregate +Input [2]: [segment#38, count#39] Keys [1]: [segment#38] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#41] -Results [3]: [segment#38, count(1)#41 AS num_customers#42, (segment#38 * 50) AS segment_base#43] -(64) TakeOrderedAndProject -Input [3]: [segment#38, num_customers#42, segment_base#43] -Arguments: 100, [segment#38 ASC NULLS FIRST, num_customers#42 ASC NULLS FIRST], [segment#38, num_customers#42, segment_base#43] +(57) CometTakeOrderedAndProject +Input [3]: [segment#38, num_customers#40, segment_base#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#38 ASC NULLS FIRST,num_customers#40 ASC NULLS FIRST], output=[segment#38,num_customers#40,segment_base#41]), [segment#38, num_customers#40, segment_base#41], 100, 0, [segment#38 ASC NULLS FIRST, num_customers#40 ASC NULLS FIRST], [segment#38, num_customers#40, segment_base#41] + +(58) CometColumnarToRow [codegen id : 1] +Input [3]: [segment#38, num_customers#40, segment_base#41] ===== Subqueries ===== -Subquery:1 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* HashAggregate (71) -+- Exchange (70) - +- * HashAggregate (69) - +- * Project (68) - +- * Filter (67) - +- * ColumnarToRow (66) - +- Scan parquet spark_catalog.default.date_dim (65) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#33, [id=#34] +* CometColumnarToRow (65) ++- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometNativeScan parquet spark_catalog.default.date_dim (59) -(65) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#44, d_year#45, d_moy#46] +(59) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#42, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(66) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] - -(67) Filter [codegen id : 1] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Condition : (((isnotnull(d_year#45) AND isnotnull(d_moy#46)) AND (d_year#45 = 1998)) AND (d_moy#46 = 12)) +(60) CometFilter +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Condition : (((isnotnull(d_year#17) AND isnotnull(d_moy#18)) AND (d_year#17 = 1998)) AND (d_moy#18 = 12)) -(68) Project [codegen id : 1] -Output [1]: [(d_month_seq#44 + 1) AS (d_month_seq + 1)#47] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] +(61) CometProject +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Arguments: [(d_month_seq + 1)#43], [(d_month_seq#42 + 1) AS (d_month_seq + 1)#43] -(69) HashAggregate [codegen id : 1] -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] +(62) CometHashAggregate +Input [1]: [(d_month_seq + 1)#43] +Keys [1]: [(d_month_seq + 1)#43] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 1)#47] -(70) Exchange -Input [1]: [(d_month_seq + 1)#47] -Arguments: hashpartitioning((d_month_seq + 1)#47, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(63) CometExchange +Input [1]: [(d_month_seq + 1)#43] +Arguments: hashpartitioning((d_month_seq + 1)#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(71) HashAggregate [codegen id : 2] -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] +(64) CometHashAggregate +Input [1]: [(d_month_seq + 1)#43] +Keys [1]: [(d_month_seq + 1)#43] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 1)#47] -Subquery:2 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#33, [id=#34] -* HashAggregate (78) -+- Exchange (77) - +- * HashAggregate (76) - +- * Project (75) - +- * Filter (74) - +- * ColumnarToRow (73) - +- Scan parquet spark_catalog.default.date_dim (72) +(65) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#43] + +Subquery:2 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#35, [id=#36] +* CometColumnarToRow (72) ++- CometHashAggregate (71) + +- CometExchange (70) + +- CometHashAggregate (69) + +- CometProject (68) + +- CometFilter (67) + +- CometNativeScan parquet spark_catalog.default.date_dim (66) -(72) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#48, d_year#49, d_moy#50] +(66) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#42, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] +(67) CometFilter +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Condition : (((isnotnull(d_year#17) AND isnotnull(d_moy#18)) AND (d_year#17 = 1998)) AND (d_moy#18 = 12)) -(74) Filter [codegen id : 1] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] -Condition : (((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 1998)) AND (d_moy#50 = 12)) +(68) CometProject +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Arguments: [(d_month_seq + 3)#44], [(d_month_seq#42 + 3) AS (d_month_seq + 3)#44] -(75) Project [codegen id : 1] -Output [1]: [(d_month_seq#48 + 3) AS (d_month_seq + 3)#51] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] - -(76) HashAggregate [codegen id : 1] -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] +(69) CometHashAggregate +Input [1]: [(d_month_seq + 3)#44] +Keys [1]: [(d_month_seq + 3)#44] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 3)#51] -(77) Exchange -Input [1]: [(d_month_seq + 3)#51] -Arguments: hashpartitioning((d_month_seq + 3)#51, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(70) CometExchange +Input [1]: [(d_month_seq + 3)#44] +Arguments: hashpartitioning((d_month_seq + 3)#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(78) HashAggregate [codegen id : 2] -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] +(71) CometHashAggregate +Input [1]: [(d_month_seq + 3)#44] +Keys [1]: [(d_month_seq + 3)#44] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 3)#51] + +(72) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#44] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/extended.txt new file mode 100644 index 0000000000..f51f7f1fe6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/extended.txt @@ -0,0 +1,76 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometUnion + : : : : : : : :- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometProject + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : :- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- Subquery + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 69 out of 71 eligible operators (97%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/simplified.txt index 8f25066aa4..65164c2ea4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/simplified.txt @@ -1,121 +1,80 @@ -TakeOrderedAndProject [segment,num_customers,segment_base] - WholeStageCodegen (13) - HashAggregate [segment,count] [count(1),num_customers,segment_base,count] - InputAdapter - Exchange [segment] #1 - WholeStageCodegen (12) - HashAggregate [segment] [count,count] - HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum] - InputAdapter - Exchange [c_customer_sk] #2 - WholeStageCodegen (11) - HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum] - Project [c_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ca_county,ca_state,s_county,s_state] - Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - HashAggregate [c_customer_sk,c_current_addr_sk] - InputAdapter - Exchange [c_customer_sk,c_current_addr_sk] #3 - WholeStageCodegen (6) - HashAggregate [c_customer_sk,c_current_addr_sk] - Project [c_customer_sk,c_current_addr_sk] - BroadcastHashJoin [customer_sk,c_customer_sk] - Project [customer_sk] - BroadcastHashJoin [sold_date_sk,d_date_sk] - Project [sold_date_sk,customer_sk] - BroadcastHashJoin [item_sk,i_item_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - WholeStageCodegen (2) - Project [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [i_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [segment,num_customers,segment_base] + CometHashAggregate [count] [segment,num_customers,segment_base,count(1)] + CometExchange [segment] #1 + CometHashAggregate [segment,count] + CometHashAggregate [c_customer_sk,sum] [segment,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [c_customer_sk] #2 + CometHashAggregate [ss_ext_sales_price] [c_customer_sk,sum] + CometProject [c_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state,s_county,s_state] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk,ca_address_sk,ca_county,ca_state] + CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometExchange [c_customer_sk,c_current_addr_sk] #3 + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometProject [c_customer_sk,c_current_addr_sk] + CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] + CometProject [customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,d_date_sk] + CometProject [sold_date_sk,customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,item_sk,i_item_sk] + CometUnion [sold_date_sk,customer_sk,item_sk] + CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk] #4 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 + CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state] + CometFilter [ca_address_sk,ca_county,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometBroadcastExchange [s_county,s_state] #9 + CometProject [s_state] [s_county,s_state] + CometFilter [s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_county,s_state] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_address_sk,ca_county,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Filter [s_county,s_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_county,s_state] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (10) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [(d_month_seq + 1)] - InputAdapter - Exchange [(d_month_seq + 1)] #11 - WholeStageCodegen (1) - HashAggregate [(d_month_seq + 1)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - Subquery #2 - WholeStageCodegen (2) - HashAggregate [(d_month_seq + 3)] - InputAdapter - Exchange [(d_month_seq + 3)] #12 - WholeStageCodegen (1) - HashAggregate [(d_month_seq + 3)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometHashAggregate [(d_month_seq + 1)] + CometExchange [(d_month_seq + 1)] #11 + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometExchange [(d_month_seq + 3)] #12 + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/explain.txt index ea07fa708f..db0a08d421 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/explain.txt @@ -1,125 +1,111 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.item (9) + + +(1) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [1]: [d_date_sk#1] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [1]: [d_date_sk#1] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] +(14) CometProject +Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(18) HashAggregate [codegen id : 3] -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [3]: [i_brand#9, i_brand_id#8, sum#12] -(19) Exchange -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Keys [2]: [i_brand#9, i_brand_id#8] +(17) CometHashAggregate +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] -(21) TakeOrderedAndProject -Input [3]: [brand_id#14, brand#15, ext_price#16] -Arguments: 100, [ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [brand_id#14, brand#15, ext_price#16] +(18) CometTakeOrderedAndProject +Input [3]: [brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[brand_id#13,brand#14,ext_price#15]), [brand_id#13, brand#14, ext_price#15], 100, 0, [ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [brand_id#13, brand#14, ext_price#15] + +(19) CometColumnarToRow [codegen id : 1] +Input [3]: [brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/extended.txt new file mode 100644 index 0000000000..e09de16f3a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/extended.txt @@ -0,0 +1,21 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 18 out of 18 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/simplified.txt index c5eb0a9649..a2adacc531 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [ext_price,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [brand_id,brand,ext_price] + CometHashAggregate [sum] [brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/explain.txt index 66859a80d7..d6176f5bd9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/explain.txt @@ -1,391 +1,348 @@ == Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +(8) CometProject Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] +(11) CometProject Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +(14) CometProject Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(18) Scan parquet spark_catalog.default.item +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] - -(20) Filter [codegen id : 4] +(16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(21) Scan parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_color#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_color, [blanched ,burnished ,slate ])] ReadSchema: struct -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#12, i_color#13] - -(23) Filter [codegen id : 3] +(18) CometFilter Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) +Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#13, 20, true, false, true) IN (slate ,blanched ,burnished ) -(24) Project [codegen id : 3] -Output [1]: [i_item_id#12] +(19) CometProject Input [2]: [i_item_id#12, i_color#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] -(25) BroadcastExchange -Input [1]: [i_item_id#12] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] +(20) CometBroadcastExchange +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_id#11] -Right keys [1]: [i_item_id#12] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(27) BroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_item_id#11] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(30) HashAggregate [codegen id : 5] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] -(31) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_item_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_item_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(46) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] + +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] + +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] + +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight + +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] + +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] + +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] + +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometHashAggregate +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] + +(43) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_item_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_item_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_item_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_item_id#37] -Keys [1]: [i_item_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_item_id#37, sum#39] - -(61) Exchange -Input [2]: [i_item_id#37, sum#39] -Arguments: hashpartitioning(i_item_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#37, sum#39] -Keys [1]: [i_item_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_item_id#11, sum#44, isEmpty#45] - -(65) Exchange -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(66) HashAggregate [codegen id : 20] -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_item_id#11, sum(total_sales#17)#46 AS total_sales#47] - -(67) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#47] -Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_item_id#11, total_sales#47] +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/extended.txt new file mode 100644 index 0000000000..7d3fb6fd9a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/extended.txt @@ -0,0 +1,92 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 89 out of 89 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/simplified.txt index 63e5d1c484..7cfa14025c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/simplified.txt @@ -1,101 +1,64 @@ -TakeOrderedAndProject [total_sales,i_item_id] - WholeStageCodegen (20) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (19) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_item_id] - Filter [i_color] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_id,i_color] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (11) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,total_sales] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] + CometExchange [i_item_id] #1 + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] + CometUnion [i_item_id,total_sales] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_color] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_color] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/explain.txt index eef23bdd3c..1bc39b2ef4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.call_center (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(4) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) -(6) Filter [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_call_center_sk#4] -Right keys [1]: [cc_call_center_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] -(21) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] -(27) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] -(30) Filter [codegen id : 22] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] + +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] -(32) ReusedExchange [Reuses operator id: 23] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(33) HashAggregate [codegen id : 12] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#16] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(36) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#16 AS sum_sales#28, rn#27] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16, rn#27] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(38) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] + +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(41) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#34], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#16 AS sum_sales#35, rn#34] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16, rn#34] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=8] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#34 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] +(47) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/extended.txt new file mode 100644 index 0000000000..357307b654 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.call_center + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/simplified.txt index 7a8378ca00..1a3c0e7f18 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,cc_name] #1 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,cc_name] #7 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/explain.txt index 6cefb2c4a1..039db87261 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/explain.txt @@ -1,373 +1,387 @@ == Physical Plan == -TakeOrderedAndProject (59) -+- * Project (58) - +- * BroadcastHashJoin Inner BuildRight (57) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Filter (26) - : : +- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * BroadcastHashJoin LeftSemi BuildRight (18) - : : :- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (17) - : : +- * Project (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) - : : +- Scan parquet spark_catalog.default.date_dim (13) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * HashAggregate (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Filter (29) - : : : +- * ColumnarToRow (28) - : : : +- Scan parquet spark_catalog.default.catalog_sales (27) - : : +- ReusedExchange (30) - : +- ReusedExchange (33) - +- BroadcastExchange (56) - +- * Filter (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- * Project (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Filter (45) - : : +- * ColumnarToRow (44) - : : +- Scan parquet spark_catalog.default.web_sales (43) - : +- ReusedExchange (46) - +- ReusedExchange (49) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (63) ++- CometTakeOrderedAndProject (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometFilter (23) + : : +- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) + : : :- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (11) + : +- CometBroadcastExchange (44) + : +- CometFilter (43) + : +- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (24) + : : +- ReusedExchange (26) + : +- CometBroadcastExchange (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometFilter (30) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (29) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometNativeScan parquet spark_catalog.default.date_dim (31) + +- CometBroadcastExchange (59) + +- CometFilter (58) + +- CometHashAggregate (57) + +- CometExchange (56) + +- CometHashAggregate (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometFilter (48) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (49) + +- ReusedExchange (52) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(6) Filter [codegen id : 1] +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(7) BroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 4] -Output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5] +(8) CometProject +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#6, d_date#7] - -(12) Filter [codegen id : 3] -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(13) Scan parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 2] -Input [2]: [d_date#8, d_week_seq#9] - -(15) Filter [codegen id : 2] +(12) CometFilter Input [2]: [d_date#8, d_week_seq#9] Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery scalar-subquery#10, [id=#11])) -(16) Project [codegen id : 2] -Output [1]: [d_date#8] +(13) CometProject Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8#12], [d_date#8 AS d_date#8#12] -(17) BroadcastExchange -Input [1]: [d_date#8] -Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [plan_id=2] - -(18) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date#7] -Right keys [1]: [d_date#8] -Join type: LeftSemi -Join condition: None - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_date#7] - -(20) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [2]: [ss_ext_sales_price#2, i_item_id#5] -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, d_date_sk#6] - -(23) HashAggregate [codegen id : 4] -Input [2]: [ss_ext_sales_price#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(14) CometBroadcastExchange +Input [1]: [d_date#8#12] +Arguments: [d_date#8#12] + +(15) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#8#12] +Arguments: [d_date#8], [d_date#8#12], LeftSemi, BuildRight + +(16) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] + +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] -(24) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 15] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] +(22) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [2]: [i_item_id#5 AS item_id#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS ss_item_rev#16] -(26) Filter [codegen id : 15] -Input [2]: [item_id#15, ss_item_rev#16] -Condition : isnotnull(ss_item_rev#16) +(23) CometFilter +Input [2]: [item_id#14, ss_item_rev#15] +Condition : isnotnull(ss_item_rev#15) -(27) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +(24) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#19)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 8] -Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] - -(29) Filter [codegen id : 8] -Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Condition : isnotnull(cs_item_sk#17) - -(30) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#20, i_item_id#21] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_item_sk#17] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [3]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21] -Input [5]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_sk#20, i_item_id#21] - -(33) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#22] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#19] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 8] -Output [2]: [cs_ext_sales_price#18, i_item_id#21] -Input [4]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21, d_date_sk#22] - -(36) HashAggregate [codegen id : 8] -Input [2]: [cs_ext_sales_price#18, i_item_id#21] -Keys [1]: [i_item_id#21] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -Aggregate Attributes [1]: [sum#23] -Results [2]: [i_item_id#21, sum#24] - -(37) Exchange -Input [2]: [i_item_id#21, sum#24] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(38) HashAggregate [codegen id : 9] -Input [2]: [i_item_id#21, sum#24] -Keys [1]: [i_item_id#21] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#18))#25] -Results [2]: [i_item_id#21 AS item_id#26, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#18))#25,17,2) AS cs_item_rev#27] - -(39) Filter [codegen id : 9] -Input [2]: [item_id#26, cs_item_rev#27] -Condition : isnotnull(cs_item_rev#27) - -(40) BroadcastExchange -Input [2]: [item_id#26, cs_item_rev#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#26] -Join type: Inner -Join condition: ((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * cs_item_rev#27)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * cs_item_rev#27))) AND (cast(cs_item_rev#27 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(cs_item_rev#27 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) - -(42) Project [codegen id : 15] -Output [3]: [item_id#15, ss_item_rev#16, cs_item_rev#27] -Input [4]: [item_id#15, ss_item_rev#16, item_id#26, cs_item_rev#27] - -(43) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] +(25) CometFilter +Input [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#16) + +(26) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#19, i_item_id#20] + +(27) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [2]: [i_item_sk#19, i_item_id#20] +Arguments: [cs_item_sk#16], [i_item_sk#19], Inner, BuildRight + +(28) CometProject +Input [5]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_sk#19, i_item_id#20] +Arguments: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20], [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] + +(29) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_date#22] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#30)] +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [d_date_sk#21, d_date#22] +Condition : isnotnull(d_date_sk#21) + +(31) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq)] +ReadSchema: struct + +(32) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = ReusedSubquery Subquery scalar-subquery#10, [id=#11])) + +(33) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8], [d_date#8] + +(34) CometBroadcastExchange +Input [1]: [d_date#8] +Arguments: [d_date#8] + +(35) CometBroadcastHashJoin +Left output [2]: [d_date_sk#21, d_date#22] +Right output [1]: [d_date#8] +Arguments: [d_date#22], [d_date#8], LeftSemi, BuildRight + +(36) CometProject +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(37) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(38) CometBroadcastHashJoin +Left output [3]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#18], [d_date_sk#21], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20, d_date_sk#21] +Arguments: [cs_ext_sales_price#17, i_item_id#20], [cs_ext_sales_price#17, i_item_id#20] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_item_id#20] +Keys [1]: [i_item_id#20] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] + +(41) CometExchange +Input [2]: [i_item_id#20, sum#23] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometHashAggregate +Input [2]: [i_item_id#20, sum#23] +Keys [1]: [i_item_id#20] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] + +(43) CometFilter +Input [2]: [item_id#24, cs_item_rev#25] +Condition : isnotnull(cs_item_rev#25) + +(44) CometBroadcastExchange +Input [2]: [item_id#24, cs_item_rev#25] +Arguments: [item_id#24, cs_item_rev#25] + +(45) CometBroadcastHashJoin +Left output [2]: [item_id#14, ss_item_rev#15] +Right output [2]: [item_id#24, cs_item_rev#25] +Arguments: [item_id#14], [item_id#24], Inner, ((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * cs_item_rev#25)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * cs_item_rev#25))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ss_item_rev#15))), BuildRight + +(46) CometProject +Input [4]: [item_id#14, ss_item_rev#15, item_id#24, cs_item_rev#25] +Arguments: [item_id#14, ss_item_rev#15, cs_item_rev#25], [item_id#14, ss_item_rev#15, cs_item_rev#25] + +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 13] -Input [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] - -(45) Filter [codegen id : 13] -Input [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] -Condition : isnotnull(ws_item_sk#28) - -(46) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#31, i_item_id#32] - -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_item_sk#28] -Right keys [1]: [i_item_sk#31] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 13] -Output [3]: [ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_id#32] -Input [5]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_sk#31, i_item_id#32] - -(49) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#33] - -(50) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#30] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 13] -Output [2]: [ws_ext_sales_price#29, i_item_id#32] -Input [4]: [ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_id#32, d_date_sk#33] - -(52) HashAggregate [codegen id : 13] -Input [2]: [ws_ext_sales_price#29, i_item_id#32] -Keys [1]: [i_item_id#32] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#29))] -Aggregate Attributes [1]: [sum#34] -Results [2]: [i_item_id#32, sum#35] - -(53) Exchange -Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 14] -Input [2]: [i_item_id#32, sum#35] -Keys [1]: [i_item_id#32] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#29))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#29))#36] -Results [2]: [i_item_id#32 AS item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#29))#36,17,2) AS ws_item_rev#38] - -(55) Filter [codegen id : 14] -Input [2]: [item_id#37, ws_item_rev#38] -Condition : isnotnull(ws_item_rev#38) - -(56) BroadcastExchange -Input [2]: [item_id#37, ws_item_rev#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=8] - -(57) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#37] -Join type: Inner -Join condition: ((((((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * ws_item_rev#38)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(cs_item_rev#27 as decimal(19,3)) >= (0.9 * ws_item_rev#38))) AND (cast(cs_item_rev#27 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * cs_item_rev#27))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * cs_item_rev#27))) - -(58) Project [codegen id : 15] -Output [8]: [item_id#15, ss_item_rev#16, (((ss_item_rev#16 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS ss_dev#39, cs_item_rev#27, (((cs_item_rev#27 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS cs_dev#40, ws_item_rev#38, (((ws_item_rev#38 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS ws_dev#41, (((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38) / 3) AS average#42] -Input [5]: [item_id#15, ss_item_rev#16, cs_item_rev#27, item_id#37, ws_item_rev#38] - -(59) TakeOrderedAndProject -Input [8]: [item_id#15, ss_item_rev#16, ss_dev#39, cs_item_rev#27, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] -Arguments: 100, [item_id#15 ASC NULLS FIRST, ss_item_rev#16 ASC NULLS FIRST], [item_id#15, ss_item_rev#16, ss_dev#39, cs_item_rev#27, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] +(48) CometFilter +Input [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#26) + +(49) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#29, i_item_id#30] + +(50) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +Right output [2]: [i_item_sk#29, i_item_id#30] +Arguments: [ws_item_sk#26], [i_item_sk#29], Inner, BuildRight + +(51) CometProject +Input [5]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_sk#29, i_item_id#30] +Arguments: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30], [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30] + +(52) ReusedExchange [Reuses operator id: 37] +Output [1]: [d_date_sk#31] + +(53) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30] +Right output [1]: [d_date_sk#31] +Arguments: [ws_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight + +(54) CometProject +Input [4]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30, d_date_sk#31] +Arguments: [ws_ext_sales_price#27, i_item_id#30], [ws_ext_sales_price#27, i_item_id#30] + +(55) CometHashAggregate +Input [2]: [ws_ext_sales_price#27, i_item_id#30] +Keys [1]: [i_item_id#30] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#27))] + +(56) CometExchange +Input [2]: [i_item_id#30, sum#32] +Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(57) CometHashAggregate +Input [2]: [i_item_id#30, sum#32] +Keys [1]: [i_item_id#30] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#27))] + +(58) CometFilter +Input [2]: [item_id#33, ws_item_rev#34] +Condition : isnotnull(ws_item_rev#34) + +(59) CometBroadcastExchange +Input [2]: [item_id#33, ws_item_rev#34] +Arguments: [item_id#33, ws_item_rev#34] + +(60) CometBroadcastHashJoin +Left output [3]: [item_id#14, ss_item_rev#15, cs_item_rev#25] +Right output [2]: [item_id#33, ws_item_rev#34] +Arguments: [item_id#14], [item_id#33], Inner, ((((((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * ws_item_rev#34)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ws_item_rev#34))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * ss_item_rev#15))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * cs_item_rev#25))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * cs_item_rev#25))), BuildRight + +(61) CometProject +Input [5]: [item_id#14, ss_item_rev#15, cs_item_rev#25, item_id#33, ws_item_rev#34] +Arguments: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], [item_id#14, ss_item_rev#15, (((ss_item_rev#15 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS ss_dev#35, cs_item_rev#25, (((cs_item_rev#25 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS cs_dev#36, ws_item_rev#34, (((ws_item_rev#34 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS ws_dev#37, (((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34) / 3) AS average#38] + +(62) CometTakeOrderedAndProject +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#14 ASC NULLS FIRST,ss_item_rev#15 ASC NULLS FIRST], output=[item_id#14,ss_item_rev#15,ss_dev#35,cs_item_rev#25,cs_dev#36,ws_item_rev#34,ws_dev#37,average#38]), [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], 100, 0, [item_id#14 ASC NULLS FIRST, ss_item_rev#15 ASC NULLS FIRST], [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] + +(63) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] ===== Subqueries ===== -Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* Project (63) -+- * Filter (62) - +- * ColumnarToRow (61) - +- Scan parquet spark_catalog.default.date_dim (60) +Subquery:1 Hosting operator id = 12 Hosting Expression = Subquery scalar-subquery#10, [id=#11] +* CometColumnarToRow (67) ++- CometProject (66) + +- CometFilter (65) + +- CometNativeScan parquet spark_catalog.default.date_dim (64) -(60) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#43, d_week_seq#44] +(64) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 1] -Input [2]: [d_date#43, d_week_seq#44] +(65) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : (isnotnull(d_date#8) AND (d_date#8 = 2000-01-03)) + +(66) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_week_seq#9], [d_week_seq#9] -(62) Filter [codegen id : 1] -Input [2]: [d_date#43, d_week_seq#44] -Condition : (isnotnull(d_date#43) AND (d_date#43 = 2000-01-03)) +(67) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#9] -(63) Project [codegen id : 1] -Output [1]: [d_week_seq#44] -Input [2]: [d_date#43, d_week_seq#44] +Subquery:2 Hosting operator id = 32 Hosting Expression = ReusedSubquery Subquery scalar-subquery#10, [id=#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/extended.txt new file mode 100644 index 0000000000..30ed8737b7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/extended.txt @@ -0,0 +1,86 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- ReusedSubquery + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : +- ReusedSubquery + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 79 out of 82 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/simplified.txt index ce068388c7..eb76833038 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/simplified.txt @@ -1,93 +1,73 @@ -TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] - WholeStageCodegen (15) - Project [item_id,ss_item_rev,cs_item_rev,ws_item_rev] - BroadcastHashJoin [item_id,item_id,ss_item_rev,ws_item_rev,cs_item_rev] - Project [item_id,ss_item_rev,cs_item_rev] - BroadcastHashJoin [item_id,item_id,ss_item_rev,cs_item_rev] - Filter [ss_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),item_id,ss_item_rev,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Project [d_date_sk] - BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date] - Filter [d_week_seq] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - Filter [cs_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum] - InputAdapter - Exchange [i_item_id] #6 - WholeStageCodegen (8) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (14) - Filter [ws_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (13) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] + CometProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] + CometBroadcastHashJoin [item_id,ss_item_rev,cs_item_rev,item_id,ws_item_rev] + CometProject [item_id,ss_item_rev,cs_item_rev] + CometBroadcastHashJoin [item_id,ss_item_rev,item_id,cs_item_rev] + CometFilter [item_id,ss_item_rev] + CometHashAggregate [sum] [item_id,ss_item_rev,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id] #2 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #4 + CometProject [d_date] [d_date] + CometFilter [d_date,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cs_item_rev] #5 + CometFilter [item_id,cs_item_rev] + CometHashAggregate [sum] [item_id,cs_item_rev,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id] #6 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #8 + CometProject [d_date] + CometFilter [d_date,d_week_seq] + ReusedSubquery [d_week_seq] #1 + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,ws_item_rev] #9 + CometFilter [item_id,ws_item_rev] + CometHashAggregate [sum] [item_id,ws_item_rev,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #10 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/explain.txt index ee3a0870f0..d25b1ea057 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/explain.txt @@ -1,256 +1,239 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (18) - : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * HashAggregate (12) - : : : +- Exchange (11) - : : : +- * HashAggregate (10) - : : : +- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) - : : +- Scan parquet spark_catalog.default.store (13) - : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) - : +- Scan parquet spark_catalog.default.date_dim (19) - +- BroadcastExchange (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * HashAggregate (27) - : : +- ReusedExchange (26) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.store (28) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.date_dim (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (42) ++- CometTakeOrderedAndProject (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.store (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.date_dim (18) + +- CometBroadcastExchange (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometHashAggregate (25) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.store (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.date_dim (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) -(7) BroadcastExchange +(5) CometProject Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7], [d_date_sk#4, d_week_seq#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#6, 9, true, false, true) AS d_day_name#7] + +(6) CometBroadcastExchange +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7] -(8) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(9) Project [codegen id : 2] -Output [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] +(8) CometProject +Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] -(10) HashAggregate [codegen id : 2] -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(9) CometHashAggregate +Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Results [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(11) Exchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] +Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) HashAggregate [codegen id : 10] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] +(11) CometHashAggregate +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(13) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] +(12) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] +(13) CometFilter +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Condition : (isnotnull(s_store_sk#15) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true))) -(15) Filter [codegen id : 3] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) +(14) CometProject +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#18, s_store_name#17] -(16) BroadcastExchange -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(15) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17] -(17) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#35] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25] +Right output [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight -(18) Project [codegen id : 10] -Output [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37] -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#35, s_store_id#36, s_store_name#37] +(17) CometProject +Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] -(19) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#38, d_week_seq#39] +(18) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 4] -Input [2]: [d_month_seq#38, d_week_seq#39] - -(21) Filter [codegen id : 4] -Input [2]: [d_month_seq#38, d_week_seq#39] -Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= 1212)) AND (d_month_seq#38 <= 1223)) AND isnotnull(d_week_seq#39)) +(19) CometFilter +Input [2]: [d_month_seq#26, d_week_seq#27] +Condition : (((isnotnull(d_month_seq#26) AND (d_month_seq#26 >= 1212)) AND (d_month_seq#26 <= 1223)) AND isnotnull(d_week_seq#27)) -(22) Project [codegen id : 4] -Output [1]: [d_week_seq#39] -Input [2]: [d_month_seq#38, d_week_seq#39] +(20) CometProject +Input [2]: [d_month_seq#26, d_week_seq#27] +Arguments: [d_week_seq#27], [d_week_seq#27] -(23) BroadcastExchange -Input [1]: [d_week_seq#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [1]: [d_week_seq#27] +Arguments: [d_week_seq#27] -(24) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#39] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [10]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] +Right output [1]: [d_week_seq#27] +Arguments: [d_week_seq#5], [d_week_seq#27], Inner, BuildRight -(25) Project [codegen id : 10] -Output [10]: [s_store_name#37 AS s_store_name1#40, d_week_seq#5 AS d_week_seq1#41, s_store_id#36 AS s_store_id1#42, sun_sales#28 AS sun_sales1#43, mon_sales#29 AS mon_sales1#44, tue_sales#30 AS tue_sales1#45, wed_sales#31 AS wed_sales1#46, thu_sales#32 AS thu_sales1#47, fri_sales#33 AS fri_sales1#48, sat_sales#34 AS sat_sales1#49] -Input [11]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37, d_week_seq#39] +(23) CometProject +Input [11]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17, d_week_seq#27] +Arguments: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37], [s_store_name#17 AS s_store_name1#28, d_week_seq#5 AS d_week_seq1#29, s_store_id#18 AS s_store_id1#30, sun_sales#19 AS sun_sales1#31, mon_sales#20 AS mon_sales1#32, tue_sales#21 AS tue_sales1#33, wed_sales#22 AS wed_sales1#34, thu_sales#23 AS thu_sales1#35, fri_sales#24 AS fri_sales1#36, sat_sales#25 AS sat_sales1#37] -(26) ReusedExchange [Reuses operator id: 11] -Output [9]: [d_week_seq#5, ss_store_sk#1, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(24) ReusedExchange [Reuses operator id: 10] +Output [9]: [d_week_seq#5, ss_store_sk#1, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] -(27) HashAggregate [codegen id : 9] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(25) CometHashAggregate +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(28) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#57, s_store_id#58] +(26) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_store_id#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#57, s_store_id#58] +(27) CometFilter +Input [2]: [s_store_sk#45, s_store_id#46] +Condition : (isnotnull(s_store_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#46, 16, true, false, true))) -(30) Filter [codegen id : 7] -Input [2]: [s_store_sk#57, s_store_id#58] -Condition : (isnotnull(s_store_sk#57) AND isnotnull(s_store_id#58)) +(28) CometProject +Input [2]: [s_store_sk#45, s_store_id#46] +Arguments: [s_store_sk#45, s_store_id#47], [s_store_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#46, 16, true, false, true) AS s_store_id#47] -(31) BroadcastExchange -Input [2]: [s_store_sk#57, s_store_id#58] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [2]: [s_store_sk#45, s_store_id#47] +Arguments: [s_store_sk#45, s_store_id#47] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#57] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25] +Right output [2]: [s_store_sk#45, s_store_id#47] +Arguments: [ss_store_sk#1], [s_store_sk#45], Inner, BuildRight -(33) Project [codegen id : 9] -Output [9]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#58] -Input [11]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#57, s_store_id#58] +(31) CometProject +Input [11]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#45, s_store_id#47] +Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47] -(34) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#59, d_week_seq#60] +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#48, d_week_seq#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [2]: [d_month_seq#59, d_week_seq#60] +(33) CometFilter +Input [2]: [d_month_seq#48, d_week_seq#49] +Condition : (((isnotnull(d_month_seq#48) AND (d_month_seq#48 >= 1224)) AND (d_month_seq#48 <= 1235)) AND isnotnull(d_week_seq#49)) -(36) Filter [codegen id : 8] -Input [2]: [d_month_seq#59, d_week_seq#60] -Condition : (((isnotnull(d_month_seq#59) AND (d_month_seq#59 >= 1224)) AND (d_month_seq#59 <= 1235)) AND isnotnull(d_week_seq#60)) +(34) CometProject +Input [2]: [d_month_seq#48, d_week_seq#49] +Arguments: [d_week_seq#49], [d_week_seq#49] -(37) Project [codegen id : 8] -Output [1]: [d_week_seq#60] -Input [2]: [d_month_seq#59, d_week_seq#60] +(35) CometBroadcastExchange +Input [1]: [d_week_seq#49] +Arguments: [d_week_seq#49] -(38) BroadcastExchange -Input [1]: [d_week_seq#60] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(36) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47] +Right output [1]: [d_week_seq#49] +Arguments: [d_week_seq#5], [d_week_seq#49], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#60] -Join type: Inner -Join condition: None +(37) CometProject +Input [10]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47, d_week_seq#49] +Arguments: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58], [d_week_seq#5 AS d_week_seq2#50, s_store_id#47 AS s_store_id2#51, sun_sales#19 AS sun_sales2#52, mon_sales#20 AS mon_sales2#53, tue_sales#21 AS tue_sales2#54, wed_sales#22 AS wed_sales2#55, thu_sales#23 AS thu_sales2#56, fri_sales#24 AS fri_sales2#57, sat_sales#25 AS sat_sales2#58] -(40) Project [codegen id : 9] -Output [9]: [d_week_seq#5 AS d_week_seq2#61, s_store_id#58 AS s_store_id2#62, sun_sales#28 AS sun_sales2#63, mon_sales#29 AS mon_sales2#64, tue_sales#30 AS tue_sales2#65, wed_sales#31 AS wed_sales2#66, thu_sales#32 AS thu_sales2#67, fri_sales#33 AS fri_sales2#68, sat_sales#34 AS sat_sales2#69] -Input [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#58, d_week_seq#60] +(38) CometBroadcastExchange +Input [9]: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] +Arguments: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] -(41) BroadcastExchange -Input [9]: [d_week_seq2#61, s_store_id2#62, sun_sales2#63, mon_sales2#64, tue_sales2#65, wed_sales2#66, thu_sales2#67, fri_sales2#68, sat_sales2#69] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [plan_id=7] +(39) CometBroadcastHashJoin +Left output [10]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37] +Right output [9]: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] +Arguments: [s_store_id1#30, d_week_seq1#29], [s_store_id2#51, (d_week_seq2#50 - 52)], Inner, BuildRight -(42) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [s_store_id1#42, d_week_seq1#41] -Right keys [2]: [s_store_id2#62, (d_week_seq2#61 - 52)] -Join type: Inner -Join condition: None +(40) CometProject +Input [19]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37, d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] +Arguments: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1#31 / sun_sales2#52) AS (sun_sales1 / sun_sales2)#59, (mon_sales1#32 / mon_sales2#53) AS (mon_sales1 / mon_sales2)#60, (tue_sales1#33 / tue_sales2#54) AS (tue_sales1 / tue_sales2)#61, (wed_sales1#34 / wed_sales2#55) AS (wed_sales1 / wed_sales2)#62, (thu_sales1#35 / thu_sales2#56) AS (thu_sales1 / thu_sales2)#63, (fri_sales1#36 / fri_sales2#57) AS (fri_sales1 / fri_sales2)#64, (sat_sales1#37 / sat_sales2#58) AS (sat_sales1 / sat_sales2)#65] -(43) Project [codegen id : 10] -Output [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1#43 / sun_sales2#63) AS (sun_sales1 / sun_sales2)#70, (mon_sales1#44 / mon_sales2#64) AS (mon_sales1 / mon_sales2)#71, (tue_sales1#45 / tue_sales2#65) AS (tue_sales1 / tue_sales2)#72, (wed_sales1#46 / wed_sales2#66) AS (wed_sales1 / wed_sales2)#73, (thu_sales1#47 / thu_sales2#67) AS (thu_sales1 / thu_sales2)#74, (fri_sales1#48 / fri_sales2#68) AS (fri_sales1 / fri_sales2)#75, (sat_sales1#49 / sat_sales2#69) AS (sat_sales1 / sat_sales2)#76] -Input [19]: [s_store_name1#40, d_week_seq1#41, s_store_id1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#61, s_store_id2#62, sun_sales2#63, mon_sales2#64, tue_sales2#65, wed_sales2#66, thu_sales2#67, fri_sales2#68, sat_sales2#69] +(41) CometTakeOrderedAndProject +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name1#28 ASC NULLS FIRST,s_store_id1#30 ASC NULLS FIRST,d_week_seq1#29 ASC NULLS FIRST], output=[s_store_name1#28,s_store_id1#30,d_week_seq1#29,(sun_sales1 / sun_sales2)#59,(mon_sales1 / mon_sales2)#60,(tue_sales1 / tue_sales2)#61,(wed_sales1 / wed_sales2)#62,(thu_sales1 / thu_sales2)#63,(fri_sales1 / fri_sales2)#64,(sat_sales1 / sat_sales2)#65]), [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65], 100, 0, [s_store_name1#28 ASC NULLS FIRST, s_store_id1#30 ASC NULLS FIRST, d_week_seq1#29 ASC NULLS FIRST], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65] -(44) TakeOrderedAndProject -Input [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#70, (mon_sales1 / mon_sales2)#71, (tue_sales1 / tue_sales2)#72, (wed_sales1 / wed_sales2)#73, (thu_sales1 / thu_sales2)#74, (fri_sales1 / fri_sales2)#75, (sat_sales1 / sat_sales2)#76] -Arguments: 100, [s_store_name1#40 ASC NULLS FIRST, s_store_id1#42 ASC NULLS FIRST, d_week_seq1#41 ASC NULLS FIRST], [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#70, (mon_sales1 / mon_sales2)#71, (tue_sales1 / tue_sales2)#72, (wed_sales1 / wed_sales2)#73, (thu_sales1 / thu_sales2)#74, (fri_sales1 / fri_sales2)#75, (sat_sales1 / sat_sales2)#76] +(42) CometColumnarToRow [codegen id : 1] +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/extended.txt new file mode 100644 index 0000000000..bfd467b72b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/extended.txt @@ -0,0 +1,53 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 50 out of 50 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/simplified.txt index 77b1718af1..72823b14c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/simplified.txt @@ -1,66 +1,44 @@ -TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] - WholeStageCodegen (10) - Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2] - Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq,ss_store_sk] #1 - WholeStageCodegen (2) - HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] + CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] + CometBroadcastHashJoin [s_store_name1,d_week_seq1,s_store_id1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometProject [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [s_store_name1,d_week_seq1,s_store_id1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name,d_week_seq] + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] + CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id,s_store_name] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometExchange [d_week_seq,ss_store_sk] #1 + CometHashAggregate [d_day_name,ss_sales_price] [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] + CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [d_week_seq] #4 + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 + CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] + CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [d_week_seq] #7 + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/explain.txt index bd5ce97614..7f46d9ba5b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/explain.txt @@ -1,302 +1,274 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Project (38) - +- * BroadcastHashJoin Inner BuildRight (37) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer_address (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store_sales (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Filter (32) - +- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.item (26) - - -(1) Scan parquet spark_catalog.default.customer_address +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometNativeScan parquet spark_catalog.default.item (22) + + +(1) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) -(3) Filter [codegen id : 7] +(3) CometProject Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] +(4) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) -(7) BroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ca_address_sk#1] -Right keys [1]: [c_current_addr_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight -(9) Project [codegen id : 7] -Output [2]: [ca_state#2, c_customer_sk#3] -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(10) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(9) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) -(12) Filter [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] -(13) BroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: Inner -Join condition: None +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(15) Project [codegen id : 7] -Output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(18) Filter [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#8] -Input [2]: [d_date_sk#8, d_month_seq#9] +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(20) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(22) Project [codegen id : 7] -Output [2]: [ca_state#2, ss_item_sk#5] -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(23) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(20) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) -(25) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(26) Scan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +(22) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(23) CometFilter +Input [2]: [i_current_price#14, i_category#15] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)) -(28) Filter [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) +(24) CometProject +Input [2]: [i_current_price#14, i_category#15] +Arguments: [i_current_price#14, i_category#16], [i_current_price#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) AS i_category#16] -(29) HashAggregate [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(25) CometHashAggregate +Input [2]: [i_current_price#14, i_category#16] Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [i_category#16, sum#19, count#20] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#14))] -(30) Exchange -Input [3]: [i_category#16, sum#19, count#20] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(26) CometExchange +Input [3]: [i_category#16, sum#17, count#18] +Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 5] -Input [3]: [i_category#16, sum#19, count#20] +(27) CometHashAggregate +Input [3]: [i_category#16, sum#17, count#18] Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#15))#21] -Results [2]: [cast((avg(UnscaledValue(i_current_price#15))#21 / 100.0) as decimal(11,6)) AS avg(i_current_price)#22, i_category#16] - -(32) Filter [codegen id : 5] -Input [2]: [avg(i_current_price)#22, i_category#16] -Condition : isnotnull(avg(i_current_price)#22) - -(33) BroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#16] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_category#14] -Right keys [1]: [i_category#16] -Join type: Inner -Join condition: (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)) - -(35) Project [codegen id : 6] -Output [1]: [i_item_sk#12] -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#22, i_category#16] - -(36) BroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#5] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [1]: [ca_state#2] -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] - -(39) HashAggregate [codegen id : 7] -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] +Functions [1]: [avg(UnscaledValue(i_current_price#14))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#19, i_category#16] +Condition : isnotnull(avg(i_current_price)#19) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [avg(i_current_price)#19, i_category#16] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#16], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight + +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#19, i_category#16] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] + +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [2]: [ca_state#2, count#24] -(40) Exchange -Input [2]: [ca_state#2, count#24] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(36) CometExchange +Input [2]: [ca_state#3, count#20] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(41) HashAggregate [codegen id : 8] -Input [2]: [ca_state#2, count#24] -Keys [1]: [ca_state#2] +(37) CometHashAggregate +Input [2]: [ca_state#3, count#20] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [2]: [ca_state#2 AS state#26, count(1)#25 AS cnt#27] -(42) Filter [codegen id : 8] -Input [2]: [state#26, cnt#27] -Condition : (cnt#27 >= 10) +(38) CometFilter +Input [2]: [state#21, cnt#22] +Condition : (cnt#22 >= 10) + +(39) CometTakeOrderedAndProject +Input [2]: [state#21, cnt#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, 0, [cnt#22 ASC NULLS FIRST], [state#21, cnt#22] -(43) TakeOrderedAndProject -Input [2]: [state#26, cnt#27] -Arguments: 100, [cnt#27 ASC NULLS FIRST], [state#26, cnt#27] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#21, cnt#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) - +- Scan parquet spark_catalog.default.date_dim (44) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(44) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#28, d_year#29, d_moy#30] +(41) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#10, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] +(42) CometFilter +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Condition : (((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2000)) AND (d_moy#24 = 1)) -(46) Filter [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] -Condition : (((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2000)) AND (d_moy#30 = 1)) +(43) CometProject +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Arguments: [d_month_seq#10], [d_month_seq#10] -(47) Project [codegen id : 1] -Output [1]: [d_month_seq#28] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(48) HashAggregate [codegen id : 1] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(44) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] -(49) Exchange -Input [1]: [d_month_seq#28] -Arguments: hashpartitioning(d_month_seq#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(45) CometExchange +Input [1]: [d_month_seq#10] +Arguments: hashpartitioning(d_month_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(50) HashAggregate [codegen id : 2] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(46) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] + +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/extended.txt new file mode 100644 index 0000000000..f6467c6fd9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/extended.txt @@ -0,0 +1,50 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 45 out of 46 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/simplified.txt index 53f3d8dd0b..3e74bb125f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/simplified.txt @@ -1,76 +1,52 @@ -TakeOrderedAndProject [cnt,state] - WholeStageCodegen (8) - Filter [cnt] - HashAggregate [ca_state,count] [count(1),state,cnt,count] - InputAdapter - Exchange [ca_state] #1 - WholeStageCodegen (7) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ca_state,ss_item_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ca_state,ss_item_sk,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Project [ca_state,c_customer_sk] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [state,cnt] + CometFilter [state,cnt] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] + CometExchange [ca_state] #1 + CometHashAggregate [ca_state,count] + CometProject [ca_state] + CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [d_month_seq] - InputAdapter - Exchange [d_month_seq] #5 - WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] - InputAdapter - Exchange [i_category] #8 - WholeStageCodegen (4) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_current_price,i_category] + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk] #6 + CometProject [i_item_sk] + CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] + CometFilter [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometBroadcastExchange [avg(i_current_price),i_category] #7 + CometFilter [avg(i_current_price),i_category] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] + CometExchange [i_category] #8 + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/explain.txt index abc1098de1..51f98a2de9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/explain.txt @@ -1,391 +1,348 @@ == Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometNativeScan parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +(8) CometProject Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] +(11) CometProject Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +(14) CometProject Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(18) Scan parquet spark_catalog.default.item +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] - -(20) Filter [codegen id : 4] +(16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(21) Scan parquet spark_catalog.default.item +(17) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_category#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music )] ReadSchema: struct -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#12, i_category#13] - -(23) Filter [codegen id : 3] +(18) CometFilter Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Music ) -(24) Project [codegen id : 3] -Output [1]: [i_item_id#12] +(19) CometProject Input [2]: [i_item_id#12, i_category#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] -(25) BroadcastExchange -Input [1]: [i_item_id#12] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] +(20) CometBroadcastExchange +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_id#11] -Right keys [1]: [i_item_id#12] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(27) BroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_item_id#11] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(30) HashAggregate [codegen id : 5] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] -(31) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_item_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_item_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(46) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] + +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] + +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] + +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight + +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] + +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] + +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] + +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometHashAggregate +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] + +(43) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_item_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_item_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_item_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_item_id#37] -Keys [1]: [i_item_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_item_id#37, sum#39] - -(61) Exchange -Input [2]: [i_item_id#37, sum#39] -Arguments: hashpartitioning(i_item_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#37, sum#39] -Keys [1]: [i_item_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_item_id#11, sum#44, isEmpty#45] - -(65) Exchange -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(66) HashAggregate [codegen id : 20] -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_item_id#11, sum(total_sales#17)#46 AS total_sales#47] - -(67) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#47] -Arguments: 100, [i_item_id#11 ASC NULLS FIRST, total_sales#47 ASC NULLS FIRST], [i_item_id#11, total_sales#47] +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#35 ASC NULLS FIRST,total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [i_item_id#35 ASC NULLS FIRST, total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/extended.txt new file mode 100644 index 0000000000..7d3fb6fd9a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/extended.txt @@ -0,0 +1,92 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 89 out of 89 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/simplified.txt index 91b533a488..99304a1eed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/simplified.txt @@ -1,101 +1,64 @@ -TakeOrderedAndProject [i_item_id,total_sales] - WholeStageCodegen (20) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (19) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_item_id] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_id,i_category] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (11) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,total_sales] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] + CometExchange [i_item_id] #1 + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] + CometUnion [i_item_id,total_sales] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_id,i_category] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/explain.txt index feacd1950b..bd3852145d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/explain.txt @@ -1,405 +1,362 @@ == Physical Plan == -* Project (71) -+- * BroadcastNestedLoopJoin Inner BuildRight (70) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Project (30) - : : : +- * BroadcastHashJoin Inner BuildRight (29) - : : : :- * Project (24) - : : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : : :- * Project (17) - : : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store (4) - : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.promotion (11) - : : : : +- BroadcastExchange (22) - : : : : +- * Project (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.date_dim (18) - : : : +- BroadcastExchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.customer (25) - : : +- BroadcastExchange (35) - : : +- * Project (34) - : : +- * Filter (33) - : : +- * ColumnarToRow (32) - : : +- Scan parquet spark_catalog.default.customer_address (31) - : +- BroadcastExchange (42) - : +- * Project (41) - : +- * Filter (40) - : +- * ColumnarToRow (39) - : +- Scan parquet spark_catalog.default.item (38) - +- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- * Project (56) - : : : +- * BroadcastHashJoin Inner BuildRight (55) - : : : :- * Project (53) - : : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : : :- * Filter (50) - : : : : : +- * ColumnarToRow (49) - : : : : : +- Scan parquet spark_catalog.default.store_sales (48) - : : : : +- ReusedExchange (51) - : : : +- ReusedExchange (54) - : : +- ReusedExchange (57) - : +- ReusedExchange (60) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales +* Project (65) ++- * BroadcastNestedLoopJoin Inner BuildRight (64) + :- * CometColumnarToRow (41) + : +- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (20) + : : : : +- CometBroadcastHashJoin (19) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (9) + : : : : +- CometBroadcastExchange (18) + : : : : +- CometProject (17) + : : : : +- CometFilter (16) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometFilter (22) + : : : +- CometNativeScan parquet spark_catalog.default.customer (21) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.item (32) + +- BroadcastExchange (63) + +- * CometColumnarToRow (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometProject (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometProject (49) + : : : +- CometBroadcastHashJoin (48) + : : : :- CometProject (46) + : : : : +- CometBroadcastHashJoin (45) + : : : : :- CometFilter (43) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (42) + : : : : +- ReusedExchange (44) + : : : +- ReusedExchange (47) + : : +- ReusedExchange (50) + : +- ReusedExchange (53) + +- ReusedExchange (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 7] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.store +(3) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#7, s_gmt_offset#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [s_store_sk#7, s_gmt_offset#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [s_store_sk#7, s_gmt_offset#8] Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) -(7) Project [codegen id : 1] -Output [1]: [s_store_sk#7] +(5) CometProject Input [2]: [s_store_sk#7, s_gmt_offset#8] +Arguments: [s_store_sk#7], [s_store_sk#7] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [s_store_sk#7] -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight -(10) Project [codegen id : 7] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +(8) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(11) Scan parquet spark_catalog.default.promotion +(9) CometNativeScan parquet spark_catalog.default.promotion Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(Or(EqualTo(p_channel_dmail,Y),EqualTo(p_channel_email,Y)),EqualTo(p_channel_tv,Y)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#10, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#11, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#12, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [p_promo_sk#9] +(11) CometProject Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Arguments: [p_promo_sk#9], [p_promo_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [p_promo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [p_promo_sk#9] -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_promo_sk#4] -Right keys [1]: [p_promo_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [p_promo_sk#9] +Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight -(17) Project [codegen id : 7] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] +(14) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -(18) Scan parquet spark_catalog.default.date_dim +(15) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) -(21) Project [codegen id : 3] -Output [1]: [d_date_sk#13] +(17) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Arguments: [d_date_sk#13], [d_date_sk#13] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#13] -(23) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight -(24) Project [codegen id : 7] -Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] +(20) CometProject Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -(25) Scan parquet spark_catalog.default.customer +(21) CometNativeScan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 4] -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] - -(27) Filter [codegen id : 4] +(22) CometFilter Input [2]: [c_customer_sk#16, c_current_addr_sk#17] Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) -(28) BroadcastExchange +(23) CometBroadcastExchange Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [c_customer_sk#16, c_current_addr_sk#17] -(29) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#16] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] +Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight -(30) Project [codegen id : 7] -Output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +(25) CometProject Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -(31) Scan parquet spark_catalog.default.customer_address +(26) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 5] -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] - -(33) Filter [codegen id : 5] +(27) CometFilter Input [2]: [ca_address_sk#18, ca_gmt_offset#19] Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) -(34) Project [codegen id : 5] -Output [1]: [ca_address_sk#18] +(28) CometProject Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] -(35) BroadcastExchange +(29) CometBroadcastExchange Input [1]: [ca_address_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#18] -(36) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#17] -Right keys [1]: [ca_address_sk#18] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +Right output [1]: [ca_address_sk#18] +Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight -(37) Project [codegen id : 7] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#5] +(31) CometProject Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] -(38) Scan parquet spark_catalog.default.item +(32) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#20, i_category#21] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 6] +(33) CometFilter Input [2]: [i_item_sk#20, i_category#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#21, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#20)) -(40) Filter [codegen id : 6] +(34) CometProject Input [2]: [i_item_sk#20, i_category#21] -Condition : ((isnotnull(i_category#21) AND (i_category#21 = Jewelry )) AND isnotnull(i_item_sk#20)) +Arguments: [i_item_sk#20], [i_item_sk#20] -(41) Project [codegen id : 6] -Output [1]: [i_item_sk#20] -Input [2]: [i_item_sk#20, i_category#21] - -(42) BroadcastExchange +(35) CometBroadcastExchange Input [1]: [i_item_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [i_item_sk#20] -(43) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight -(44) Project [codegen id : 7] -Output [1]: [ss_ext_sales_price#5] +(37) CometProject Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] +Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] -(45) HashAggregate [codegen id : 7] +(38) CometHashAggregate Input [1]: [ss_ext_sales_price#5] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#22] -Results [1]: [sum#23] -(46) Exchange -Input [1]: [sum#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(39) CometExchange +Input [1]: [sum#22] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(47) HashAggregate [codegen id : 15] -Input [1]: [sum#23] +(40) CometHashAggregate +Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#24] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#24,17,2) AS promotions#25] -(48) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] +(41) CometColumnarToRow [codegen id : 2] +Input [1]: [promotions#23] + +(42) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#30)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#28)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 13] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] +(43) CometFilter +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Condition : ((isnotnull(ss_store_sk#26) AND isnotnull(ss_customer_sk#25)) AND isnotnull(ss_item_sk#24)) -(50) Filter [codegen id : 13] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] -Condition : ((isnotnull(ss_store_sk#28) AND isnotnull(ss_customer_sk#27)) AND isnotnull(ss_item_sk#26)) +(44) ReusedExchange [Reuses operator id: 6] +Output [1]: [s_store_sk#29] -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [s_store_sk#31] +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [s_store_sk#29] +Arguments: [ss_store_sk#26], [s_store_sk#29], Inner, BuildRight -(52) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_store_sk#28] -Right keys [1]: [s_store_sk#31] -Join type: Inner -Join condition: None +(46) CometProject +Input [6]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28, s_store_sk#29] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] -(53) Project [codegen id : 13] -Output [4]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30] -Input [6]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30, s_store_sk#31] +(47) ReusedExchange [Reuses operator id: 18] +Output [1]: [d_date_sk#30] -(54) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#32] +(48) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [d_date_sk#30] +Arguments: [ss_sold_date_sk#28], [d_date_sk#30], Inner, BuildRight -(55) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_sold_date_sk#30] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None +(49) CometProject +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28, d_date_sk#30] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] -(56) Project [codegen id : 13] -Output [3]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30, d_date_sk#32] +(50) ReusedExchange [Reuses operator id: 23] +Output [2]: [c_customer_sk#31, c_current_addr_sk#32] -(57) ReusedExchange [Reuses operator id: 28] -Output [2]: [c_customer_sk#33, c_current_addr_sk#34] +(51) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] +Right output [2]: [c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_customer_sk#25], [c_customer_sk#31], Inner, BuildRight -(58) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_customer_sk#27] -Right keys [1]: [c_customer_sk#33] -Join type: Inner -Join condition: None +(52) CometProject +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32], [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] -(59) Project [codegen id : 13] -Output [3]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#34] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, c_customer_sk#33, c_current_addr_sk#34] +(53) ReusedExchange [Reuses operator id: 29] +Output [1]: [ca_address_sk#33] -(60) ReusedExchange [Reuses operator id: 35] -Output [1]: [ca_address_sk#35] +(54) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] +Right output [1]: [ca_address_sk#33] +Arguments: [c_current_addr_sk#32], [ca_address_sk#33], Inner, BuildRight -(61) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None +(55) CometProject +Input [4]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32, ca_address_sk#33] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27], [ss_item_sk#24, ss_ext_sales_price#27] -(62) Project [codegen id : 13] -Output [2]: [ss_item_sk#26, ss_ext_sales_price#29] -Input [4]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#34, ca_address_sk#35] +(56) ReusedExchange [Reuses operator id: 35] +Output [1]: [i_item_sk#34] -(63) ReusedExchange [Reuses operator id: 42] -Output [1]: [i_item_sk#36] +(57) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#24, ss_ext_sales_price#27] +Right output [1]: [i_item_sk#34] +Arguments: [ss_item_sk#24], [i_item_sk#34], Inner, BuildRight -(64) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_item_sk#26] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None +(58) CometProject +Input [3]: [ss_item_sk#24, ss_ext_sales_price#27, i_item_sk#34] +Arguments: [ss_ext_sales_price#27], [ss_ext_sales_price#27] -(65) Project [codegen id : 13] -Output [1]: [ss_ext_sales_price#29] -Input [3]: [ss_item_sk#26, ss_ext_sales_price#29, i_item_sk#36] - -(66) HashAggregate [codegen id : 13] -Input [1]: [ss_ext_sales_price#29] +(59) CometHashAggregate +Input [1]: [ss_ext_sales_price#27] Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#29))] -Aggregate Attributes [1]: [sum#37] -Results [1]: [sum#38] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))] -(67) Exchange -Input [1]: [sum#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(60) CometExchange +Input [1]: [sum#35] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(68) HashAggregate [codegen id : 14] -Input [1]: [sum#38] +(61) CometHashAggregate +Input [1]: [sum#35] Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#29))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#29))#39] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#29))#39,17,2) AS total#40] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] + +(62) CometColumnarToRow [codegen id : 1] +Input [1]: [total#36] -(69) BroadcastExchange -Input [1]: [total#40] -Arguments: IdentityBroadcastMode, [plan_id=9] +(63) BroadcastExchange +Input [1]: [total#36] +Arguments: IdentityBroadcastMode, [plan_id=3] -(70) BroadcastNestedLoopJoin [codegen id : 15] +(64) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(71) Project [codegen id : 15] -Output [3]: [promotions#25, total#40, ((cast(promotions#25 as decimal(15,4)) / cast(total#40 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#41] -Input [2]: [promotions#25, total#40] +(65) Project [codegen id : 2] +Output [3]: [promotions#23, total#36, ((cast(promotions#23 as decimal(15,4)) / cast(total#36 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#37] +Input [2]: [promotions#23, total#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/extended.txt new file mode 100644 index 0000000000..d3db57c145 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/extended.txt @@ -0,0 +1,81 @@ +Project ++- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 74 out of 77 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/simplified.txt index 86604ede26..d5eeeeebc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/simplified.txt @@ -1,103 +1,70 @@ -WholeStageCodegen (15) +WholeStageCodegen (2) Project [promotions,total] BroadcastNestedLoopJoin - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (7) - HashAggregate [ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_promo_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [s_store_sk] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [p_promo_sk] - Filter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [i_item_sk] - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category] + CometHashAggregate [sum] [promotions,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange #1 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_gmt_offset] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] + CometBroadcastExchange [p_promo_sk] #3 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter BroadcastExchange #8 - WholeStageCodegen (14) - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [s_store_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [c_customer_sk,c_current_addr_sk] #5 - InputAdapter - ReusedExchange [ca_address_sk] #6 - InputAdapter - ReusedExchange [i_item_sk] #7 + CometHashAggregate [sum] [total,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange #9 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [s_store_sk] #2 + ReusedExchange [d_date_sk] #4 + ReusedExchange [c_customer_sk,c_current_addr_sk] #5 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/explain.txt index 59701e2d06..1d9748055e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/explain.txt @@ -1,187 +1,168 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.ship_mode (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.web_site (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.web_site (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_warehouse_sk#4] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [ws_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight -(9) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] +(7) CometProject Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -(10) Scan parquet spark_catalog.default.ship_mode +(8) CometNativeScan parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [sm_ship_mode_sk#8, sm_type#9] Condition : isnotnull(sm_ship_mode_sk#8) -(13) BroadcastExchange +(10) CometProject Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] + +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_mode_sk#3] -Right keys [1]: [sm_ship_mode_sk#8] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight -(15) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] +(13) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(16) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#10, web_name#11] +(14) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#11, web_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [web_site_sk#10, web_name#11] - -(18) Filter [codegen id : 3] -Input [2]: [web_site_sk#10, web_name#11] -Condition : isnotnull(web_site_sk#10) +(15) CometFilter +Input [2]: [web_site_sk#11, web_name#12] +Condition : isnotnull(web_site_sk#11) -(19) BroadcastExchange -Input [2]: [web_site_sk#10, web_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [2]: [web_site_sk#11, web_name#12] +Arguments: [web_site_sk#11, web_name#12] -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_web_site_sk#2] -Right keys [1]: [web_site_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [web_site_sk#11, web_name#12] +Arguments: [ws_web_site_sk#2], [web_site_sk#11], Inner, BuildRight -(21) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] +(18) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_site_sk#11, web_name#12] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) -(24) Filter [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] -Input [2]: [d_date_sk#12, d_month_seq#13] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(26) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(24) CometProject +Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12, d_date_sk#13] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] -(28) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] - -(29) HashAggregate [codegen id : 5] -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(25) CometHashAggregate +Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(30) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, web_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 6] -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(32) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,web_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,web_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, web_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/extended.txt new file mode 100644 index 0000000000..52d7828b06 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.ship_mode + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/simplified.txt index 197db771f3..de80c17349 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/simplified.txt @@ -1,48 +1,31 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,web_name] #1 - WholeStageCodegen (5) - HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,w_warehouse_name] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - Project [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] - BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk] - Project [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] - BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_ship_mode_sk,ws_web_site_sk,ws_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,web_name] #1 + CometHashAggregate [ws_ship_date_sk,ws_sold_date_sk] [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] + CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] + CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [web_site_sk,web_name] #4 + CometFilter [web_site_sk,web_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/explain.txt index f3769cf4aa..a1e7286cb0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/explain.txt @@ -1,184 +1,165 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (17) - : +- * BroadcastHashJoin Inner BuildRight (16) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.store_sales (5) - : +- BroadcastExchange (15) - : +- * Project (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.date_dim (11) - +- BroadcastExchange (21) - +- * Filter (20) - +- * ColumnarToRow (19) - +- Scan parquet spark_catalog.default.store (18) - - -(1) Scan parquet spark_catalog.default.item +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,refernece ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(3) Filter [codegen id : 4] +(3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manager_id#5] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] - -(5) Scan parquet spark_catalog.default.store_sales +(4) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [i_item_sk#1, i_manager_id#5] +Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +(8) CometProject Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) -(14) Project [codegen id : 2] -Output [2]: [d_date_sk#14, d_moy#16] +(11) CometProject Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [d_date_sk#14, d_moy#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#14, d_moy#16] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_moy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(17) Project [codegen id : 4] -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +(14) CometProject Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -(18) Scan parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [s_store_sk#17] Condition : isnotnull(s_store_sk#17) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [s_store_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#17] -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight -(23) Project [codegen id : 4] -Output [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] +(19) CometProject Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] -(24) HashAggregate [codegen id : 4] +(20) CometHashAggregate Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manager_id#5, d_moy#16, sum#19] -(25) Exchange -Input [3]: [i_manager_id#5, d_moy#16, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [3]: [i_manager_id#5, d_moy#16, sum#19] +(22) CometHashAggregate +Input [3]: [i_manager_id#5, d_moy#16, sum#18] Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] -(27) Exchange -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometExchange +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [i_manager_id#5, sum_sales#19, _w0#20], [i_manager_id#5 ASC NULLS FIRST] -(28) Sort [codegen id : 6] -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] -(29) Window -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#23], [i_manager_id#5] +(26) Window +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_manager_id#5] -(30) Filter [codegen id : 7] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] -Condition : CASE WHEN (avg_monthly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_monthly_sales#23)) / avg_monthly_sales#23) > 0.1000000000000000) ELSE false END +(27) Filter [codegen id : 2] +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) ELSE false END -(31) Project [codegen id : 7] -Output [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] +(28) Project [codegen id : 2] +Output [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] -(32) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST], [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] +(29) TakeOrderedAndProject +Input [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST], [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/extended.txt new file mode 100644 index 0000000000..44fd48691b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/extended.txt @@ -0,0 +1,31 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/simplified.txt index 57a82302ae..4a2ef17f8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/simplified.txt @@ -1,49 +1,33 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [i_manager_id,sum_sales,avg_monthly_sales] Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_manager_id] - WholeStageCodegen (6) - Sort [i_manager_id] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_manager_id] #1 - WholeStageCodegen (5) - HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manager_id,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum] - Project [i_manager_id,ss_sales_price,d_moy] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_manager_id,ss_store_sk,ss_sales_price,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manager_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_moy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] + CometSort [i_manager_id,sum_sales,_w0] + CometExchange [i_manager_id] #1 + CometHashAggregate [d_moy,sum] [i_manager_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_manager_id,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_manager_id,d_moy,sum] + CometProject [i_manager_id,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manager_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_month_seq,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/explain.txt index 9dce3ef15b..e2d82c6a20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/explain.txt @@ -1,999 +1,909 @@ == Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) - :- * Sort (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- * Project (110) - : +- * BroadcastHashJoin Inner BuildRight (109) - : :- * Project (103) - : : +- * BroadcastHashJoin Inner BuildRight (102) - : : :- * Project (100) - : : : +- * BroadcastHashJoin Inner BuildRight (99) - : : : :- * Project (94) - : : : : +- * BroadcastHashJoin Inner BuildRight (93) - : : : : :- * Project (91) - : : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : : :- * Project (85) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) - : : : : : : :- * Project (82) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : : :- * Project (76) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) - : : : : : : : : :- * Project (70) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) - : : : : : : : : : :- * Project (67) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : : :- * Project (61) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : : :- * Project (52) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (51) - : : : : : : : : : : : : : :- * Project (46) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (45) - : : : : : : : : : : : : : : :- * Project (40) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (39) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- BroadcastExchange (38) - : : : : : : : : : : : : : : : +- * Filter (37) - : : : : : : : : : : : : : : : +- * ColumnarToRow (36) - : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (35) - : : : : : : : : : : : : : : +- BroadcastExchange (44) - : : : : : : : : : : : : : : +- * Filter (43) - : : : : : : : : : : : : : : +- * ColumnarToRow (42) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store (41) - : : : : : : : : : : : : : +- BroadcastExchange (50) - : : : : : : : : : : : : : +- * Filter (49) - : : : : : : : : : : : : : +- * ColumnarToRow (48) - : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.customer (47) - : : : : : : : : : : : : +- BroadcastExchange (56) - : : : : : : : : : : : : +- * Filter (55) - : : : : : : : : : : : : +- * ColumnarToRow (54) - : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (53) - : : : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : : : +- BroadcastExchange (65) - : : : : : : : : : : +- * Filter (64) - : : : : : : : : : : +- * ColumnarToRow (63) - : : : : : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (62) - : : : : : : : : : +- ReusedExchange (68) - : : : : : : : : +- BroadcastExchange (74) - : : : : : : : : +- * Filter (73) - : : : : : : : : +- * ColumnarToRow (72) - : : : : : : : : +- Scan parquet spark_catalog.default.promotion (71) - : : : : : : : +- BroadcastExchange (80) - : : : : : : : +- * Filter (79) - : : : : : : : +- * ColumnarToRow (78) - : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (77) - : : : : : : +- ReusedExchange (83) - : : : : : +- BroadcastExchange (89) - : : : : : +- * Filter (88) - : : : : : +- * ColumnarToRow (87) - : : : : : +- Scan parquet spark_catalog.default.customer_address (86) - : : : : +- ReusedExchange (92) - : : : +- BroadcastExchange (98) - : : : +- * Filter (97) - : : : +- * ColumnarToRow (96) - : : : +- Scan parquet spark_catalog.default.income_band (95) - : : +- ReusedExchange (101) - : +- BroadcastExchange (108) - : +- * Project (107) - : +- * Filter (106) - : +- * ColumnarToRow (105) - : +- Scan parquet spark_catalog.default.item (104) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * Project (123) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (122) - : : : : : : : : : : : : : : : :- * Sort (116) - : : : : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : : : : +- * Sort (121) - : : : : : : : : : : : : : : : +- * Project (120) - : : : : : : : : : : : : : : : +- * Filter (119) - : : : : : : : : : : : : : : : +- * HashAggregate (118) - : : : : : : : : : : : : : : : +- ReusedExchange (117) - : : : : : : : : : : : : : : +- BroadcastExchange (127) - : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : +- * ColumnarToRow (125) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (124) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) + : : : : : : : : : : : : : : :- CometProject (35) + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) + : : : : : : : : : : : : : : : :- CometProject (30) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) + : : : : : : : : : : : : : : : : :- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- CometSort (28) + : : : : : : : : : : : : : : : : +- CometProject (27) + : : : : : : : : : : : : : : : : +- CometFilter (26) + : : : : : : : : : : : : : : : : +- CometHashAggregate (25) + : : : : : : : : : : : : : : : : +- CometExchange (24) + : : : : : : : : : : : : : : : : +- CometHashAggregate (23) + : : : : : : : : : : : : : : : : +- CometProject (22) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) + : : : : : : : : : : : : : : : : :- CometSort (15) + : : : : : : : : : : : : : : : : : +- CometExchange (14) + : : : : : : : : : : : : : : : : : +- CometProject (13) + : : : : : : : : : : : : : : : : : +- CometFilter (12) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : +- CometSort (20) + : : : : : : : : : : : : : : : : +- CometExchange (19) + : : : : : : : : : : : : : : : : +- CometProject (18) + : : : : : : : : : : : : : : : : +- CometFilter (17) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) + : : : : : : : : : : : : : : : +- CometFilter (32) + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometNativeScan parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(6) ColumnarToRow -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(7) Filter +(5) CometFilter Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) -(8) Project -Output [2]: [sr_item_sk#13, sr_ticket_number#14] +(6) CometProject Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#13, sr_ticket_number#14] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(8) CometProject Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(11) Exchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) Sort [codegen id : 3] +(10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(13) Scan parquet spark_catalog.default.catalog_sales +(11) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(15) Filter [codegen id : 4] +(12) CometFilter Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +(13) CometProject Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -(17) Exchange +(14) CometExchange Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) Sort [codegen id : 5] +(15) CometSort Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(21) Filter [codegen id : 6] +(17) CometFilter Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(18) CometProject Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(23) Exchange +(19) CometExchange Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(24) Sort [codegen id : 7] +(20) CometSort Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_item_sk#16, cs_order_number#17] -Right keys [2]: [cr_item_sk#20, cr_order_number#21] -Join type: Inner -Join condition: None +(21) CometSortMergeJoin +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner -(26) Project [codegen id : 8] -Output [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(22) CometProject Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(27) HashAggregate [codegen id : 8] +(23) CometHashAggregate Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] Keys [1]: [cs_item_sk#16] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -(28) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(29) HashAggregate [codegen id : 9] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] +(25) CometHashAggregate +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] Keys [1]: [cs_item_sk#16] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] -(30) Filter [codegen id : 9] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) +(26) CometFilter +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(31) Project [codegen id : 9] -Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] +(27) CometProject +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] -(32) Sort [codegen id : 9] +(28) CometSort Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] -(33) SortMergeJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#16] -Join type: Inner -Join condition: None +(29) CometSortMergeJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner -(34) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(30) CometProject Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(35) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#36, d_year#37] +(31) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] +(32) CometFilter +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) -(37) Filter [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) +(33) CometBroadcastExchange +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] -(38) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(34) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None +(35) CometProject +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(40) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] - -(41) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(36) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(37) CometFilter +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(43) Filter [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) +(38) CometProject +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] -(44) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] -(45) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None +(40) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(46) Project [codegen id : 25] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] -(47) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(42) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(49) Filter [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) -(50) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(51) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] -Join type: Inner -Join condition: None +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight -(52) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(53) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#47, d_year#48] +(47) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) -(55) Filter [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(56) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight -(57) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] -Join type: Inner -Join condition: None +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] -(58) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#49, d_year#50] +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight -(60) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] -Join type: Inner -Join condition: None +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(61) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] - -(62) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +(55) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(63) ColumnarToRow [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) -(64) Filter [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] -(65) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] -(66) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight -(67) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] -(68) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] -Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(70) Project [codegen id : 25] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(71) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] +(64) CometNativeScan parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 17] -Input [1]: [p_promo_sk#55] - -(73) Filter [codegen id : 17] -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) -(74) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] -(75) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight -(76) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(77) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(69) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(78) ColumnarToRow [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(79) Filter [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(80) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight -(81) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] -Join type: Inner -Join condition: None +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(82) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(83) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(84) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] -Join type: Inner -Join condition: None +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(85) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] - -(86) Scan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(77) CometNativeScan parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(87) ColumnarToRow [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) -(88) Filter [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) +(79) CometProject +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(89) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(90) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] -Join type: Inner -Join condition: None +(81) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight -(91) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(82) CometProject +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(92) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(93) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] -Join type: Inner -Join condition: None +(84) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight -(94) Project [codegen id : 25] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(85) CometProject +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(95) Scan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +(86) CometNativeScan parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 22] -Input [1]: [ib_income_band_sk#70] - -(97) Filter [codegen id : 22] -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) -(98) BroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] -(99) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] -Join type: Inner -Join condition: None +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight -(100) Project [codegen id : 25] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(101) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#71] +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] -(102) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight -(103) Project [codegen id : 25] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(104) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(94) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(105) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(106) Filter [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) -(107) Project [codegen id : 24] -Output [2]: [i_item_sk#72, i_product_name#75] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] -(108) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] -(109) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight -(110) Project [codegen id : 25] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] -(111) HashAggregate [codegen id : 25] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -(112) HashAggregate [codegen id : 25] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] - -(113) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(114) Sort [codegen id : 26] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 - -(115) ReusedExchange [Reuses operator id: 11] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(116) Sort [codegen id : 29] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105 ASC NULLS FIRST], false, 0 - -(117) ReusedExchange [Reuses operator id: 28] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] - -(118) HashAggregate [codegen id : 35] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#34, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#35] - -(119) Filter [codegen id : 35] -Input [3]: [cs_item_sk#116, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(120) Project [codegen id : 35] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#34, refund#35] - -(121) Sort [codegen id : 35] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(124) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#124, d_year#125] + +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] + +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] + +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] + +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] + +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) + +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] + +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] + +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner + +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(113) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] - -(126) Filter [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] -Condition : ((isnotnull(d_year#125) AND (d_year#125 = 2000)) AND isnotnull(d_date_sk#124)) - -(127) BroadcastExchange -Input [2]: [d_date_sk#124, d_year#125] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -(128) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#124] -Join type: Inner -Join condition: None - -(129) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#124, d_year#125] - -(130) ReusedExchange [Reuses operator id: 44] -Output [3]: [s_store_sk#126, s_store_name#127, s_zip#128] - -(131) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#126] -Join type: Inner -Join condition: None - -(132) Project [codegen id : 51] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_sk#126, s_store_name#127, s_zip#128] - -(133) ReusedExchange [Reuses operator id: 50] -Output [6]: [c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(134) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#129] -Join type: Inner -Join condition: None - -(135) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(136) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#135, d_year#136] - -(137) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_sales_date_sk#134] -Right keys [1]: [d_date_sk#135] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134, d_date_sk#135, d_year#136] - -(139) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#137, d_year#138] - -(140) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_shipto_date_sk#133] -Right keys [1]: [d_date_sk#137] -Join type: Inner -Join condition: None - -(141) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136, d_date_sk#137, d_year#138] - -(142) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#139, cd_marital_status#140] - -(143) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#139] -Join type: Inner -Join condition: None - -(144) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_demo_sk#139, cd_marital_status#140] - -(145) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] - -(146) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_cdemo_sk#130] -Right keys [1]: [cd_demo_sk#141] -Join type: Inner -Join condition: NOT (cd_marital_status#140 = cd_marital_status#142) - -(147) Project [codegen id : 51] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140, cd_demo_sk#141, cd_marital_status#142] - -(148) ReusedExchange [Reuses operator id: 74] -Output [1]: [p_promo_sk#143] - -(149) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#143] -Join type: Inner -Join condition: None - -(150) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, p_promo_sk#143] - -(151) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#144, hd_income_band_sk#145] - -(152) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#144] -Join type: Inner -Join condition: None - -(153) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_demo_sk#144, hd_income_band_sk#145] - -(154) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] - -(155) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_hdemo_sk#131] -Right keys [1]: [hd_demo_sk#146] -Join type: Inner -Join condition: None - -(156) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_demo_sk#146, hd_income_band_sk#147] - -(157) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(158) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#148] -Join type: Inner -Join condition: None - -(159) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(160) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(161) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_addr_sk#132] -Right keys [1]: [ca_address_sk#153] -Join type: Inner -Join condition: None - -(162) Project [codegen id : 51] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(163) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#158] - -(164) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#145] -Right keys [1]: [ib_income_band_sk#158] -Join type: Inner -Join condition: None - -(165) Project [codegen id : 51] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#158] - -(166) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#159] - -(167) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#159] -Join type: Inner -Join condition: None - -(168) Project [codegen id : 51] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#159] - -(169) ReusedExchange [Reuses operator id: 108] -Output [2]: [i_item_sk#160, i_product_name#161] - -(170) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#160] -Join type: Inner -Join condition: None - -(171) Project [codegen id : 51] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] - -(172) HashAggregate [codegen id : 51] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#162, sum#163, sum#164] -Results [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] - -(173) HashAggregate [codegen id : 51] -Input [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#160 AS item_sk#168, s_store_name#127 AS store_name#169, s_zip#128 AS store_zip#170, d_year#125 AS syear#171, count(1)#84 AS cnt#172, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#173, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#174, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#175] - -(174) Exchange -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: hashpartitioning(item_sk#168, store_name#169, store_zip#170, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(175) Sort [codegen id : 52] -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: [item_sk#168 ASC NULLS FIRST, store_name#169 ASC NULLS FIRST, store_zip#170 ASC NULLS FIRST], false, 0 - -(176) SortMergeJoin [codegen id : 53] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#168, store_name#169, store_zip#170] -Join type: Inner -Join condition: (cnt#172 <= cnt#101) - -(177) Project [codegen id : 53] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] - -(178) Exchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(179) Sort [codegen id : 54] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST], true, 0 +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) + +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] + +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight + +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] + +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] + +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight + +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] + +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight + +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] + +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight + +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] + +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] + +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight + +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] + +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight + +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] + +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] + +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] + +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight + +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] + +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight + +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] + +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] + +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight + +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] + +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] + +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] + +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] + +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) + +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] + +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST] + +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/extended.txt new file mode 100644 index 0000000000..ae10068c4c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/extended.txt @@ -0,0 +1,237 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometProject + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.income_band + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 234 out of 234 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/simplified.txt index 26d89524e8..4451988479 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/simplified.txt @@ -1,270 +1,170 @@ -WholeStageCodegen (54) - Sort [product_name,store_name,cnt] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [product_name,store_name,cnt] #1 - WholeStageCodegen (53) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (26) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [ss_item_sk] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (9) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (8) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #6 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #7 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk] - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (52) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (35) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (36) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [p_promo_sk] #13 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 + CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometExchange [product_name,store_name,cnt] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #2 + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #8 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #11 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 + CometFilter [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 + CometFilter [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 + CometFilter [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #18 + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/explain.txt index 0da8f232f0..c72b9a5e75 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/explain.txt @@ -1,257 +1,229 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (26) - : +- * BroadcastHashJoin Inner BuildRight (25) - : :- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store (1) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * HashAggregate (16) - : : +- Exchange (15) - : : +- * HashAggregate (14) - : : +- * Project (13) - : : +- * BroadcastHashJoin Inner BuildRight (12) - : : :- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) - : : +- Scan parquet spark_catalog.default.date_dim (7) - : +- BroadcastExchange (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - +- BroadcastExchange (40) - +- * Filter (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.store_sales (27) - +- ReusedExchange (30) - - -(1) Scan parquet spark_catalog.default.store +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store (1) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometHashAggregate (13) + : : +- CometExchange (12) + : : +- CometHashAggregate (11) + : : +- CometProject (10) + : : +- CometBroadcastHashJoin (9) + : : :- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : +- CometBroadcastExchange (8) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (5) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (36) + +- CometFilter (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.store_sales (24) + +- ReusedExchange (26) + + +(1) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#1, s_store_name#2] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [2]: [s_store_sk#1, s_store_name#2] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(6) Filter [codegen id : 2] +(4) CometFilter Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(7) Scan parquet spark_catalog.default.date_dim +(5) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#7, d_month_seq#8] - -(9) Filter [codegen id : 1] +(6) CometFilter Input [2]: [d_date_sk#7, d_month_seq#8] Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) -(10) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(7) CometProject Input [2]: [d_date_sk#7, d_month_seq#8] +Arguments: [d_date_sk#7], [d_date_sk#7] -(11) BroadcastExchange +(8) CometBroadcastExchange Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#7] -(12) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#7] -Join type: Inner -Join condition: None +(9) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight -(13) Project [codegen id : 2] -Output [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] +(10) CometProject Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] +Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] -(14) HashAggregate [codegen id : 2] +(11) CometHashAggregate Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -(15) Exchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(12) CometExchange +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] +Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) HashAggregate [codegen id : 3] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +(13) CometHashAggregate +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#11] -Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#11,17,2) AS revenue#12] -(17) Filter [codegen id : 3] -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Condition : isnotnull(revenue#12) +(14) CometFilter +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Condition : isnotnull(revenue#10) -(18) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(15) CometBroadcastExchange +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [ss_store_sk#4, ss_item_sk#3, revenue#10] -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [s_store_sk#1] -Right keys [1]: [ss_store_sk#4] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [2]: [s_store_sk#1, s_store_name#2] +Right output [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight -(20) Project [codegen id : 9] -Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] +(17) CometProject +Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] -(21) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(18) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 4] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(19) CometFilter +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Condition : isnotnull(i_item_sk#11) -(23) Filter [codegen id : 4] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Condition : isnotnull(i_item_sk#13) +(20) CometProject +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#16] -(24) BroadcastExchange -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#3] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Right output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [ss_item_sk#3], [i_item_sk#11], Inner, BuildRight -(26) Project [codegen id : 9] -Output [7]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12, i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(23) CometProject +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10, i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] -(27) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +(24) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#21)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 6] -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] - -(29) Filter [codegen id : 6] -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Condition : isnotnull(ss_store_sk#19) - -(30) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#22] - -(31) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 6] -Output [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#22] - -(33) HashAggregate [codegen id : 6] -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum#23] -Results [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] - -(34) Exchange -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 7] -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#25] -Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#25,17,2) AS revenue#26] - -(36) HashAggregate [codegen id : 7] -Input [2]: [ss_store_sk#19, revenue#26] -Keys [1]: [ss_store_sk#19] -Functions [1]: [partial_avg(revenue#26)] -Aggregate Attributes [2]: [sum#27, count#28] -Results [3]: [ss_store_sk#19, sum#29, count#30] - -(37) Exchange -Input [3]: [ss_store_sk#19, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(38) HashAggregate [codegen id : 8] -Input [3]: [ss_store_sk#19, sum#29, count#30] -Keys [1]: [ss_store_sk#19] -Functions [1]: [avg(revenue#26)] -Aggregate Attributes [1]: [avg(revenue#26)#31] -Results [2]: [ss_store_sk#19, avg(revenue#26)#31 AS ave#32] - -(39) Filter [codegen id : 8] -Input [2]: [ss_store_sk#19, ave#32] -Condition : isnotnull(ave#32) - -(40) BroadcastExchange -Input [2]: [ss_store_sk#19, ave#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -(41) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [ss_store_sk#19] -Join type: Inner -Join condition: (cast(revenue#12 as decimal(23,7)) <= (0.1 * ave#32)) - -(42) Project [codegen id : 9] -Output [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17, ss_store_sk#19, ave#32] - -(43) TakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#14 ASC NULLS FIRST], [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(25) CometFilter +Input [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_store_sk#18) + +(26) ReusedExchange [Reuses operator id: 8] +Output [1]: [d_date_sk#21] + +(27) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(28) CometProject +Input [5]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21] +Arguments: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19], [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] + +(29) CometHashAggregate +Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] + +(30) CometExchange +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [sum(UnscaledValue(ss_sales_price#19))] + +(32) CometHashAggregate +Input [2]: [ss_store_sk#18, revenue#23] +Keys [1]: [ss_store_sk#18] +Functions [1]: [partial_avg(revenue#23)] + +(33) CometExchange +Input [3]: [ss_store_sk#18, sum#24, count#25] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [3]: [ss_store_sk#18, sum#24, count#25] +Keys [1]: [ss_store_sk#18] +Functions [1]: [avg(revenue#23)] + +(35) CometFilter +Input [2]: [ss_store_sk#18, ave#26] +Condition : isnotnull(ave#26) + +(36) CometBroadcastExchange +Input [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#18, ave#26] + +(37) CometBroadcastHashJoin +Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Right output [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#4], [ss_store_sk#18], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#26)), BuildRight + +(38) CometProject +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16, ss_store_sk#18, ave#26] +Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(39) CometTakeOrderedAndProject +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#16]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(40) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/extended.txt new file mode 100644 index 0000000000..670d8ea5e8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/simplified.txt index 6f9a1fee81..5ae7059ffe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/simplified.txt @@ -1,64 +1,42 @@ -TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - WholeStageCodegen (9) - Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave] - Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [s_store_name,ss_store_sk,ss_item_sk,revenue] - BroadcastHashJoin [s_store_sk,ss_store_sk] - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] - InputAdapter - BroadcastExchange #1 - WholeStageCodegen (3) - Filter [revenue] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #2 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_store_sk,ss_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Filter [ave] - HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] - InputAdapter - Exchange [ss_store_sk] #6 - WholeStageCodegen (7) - HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #7 - WholeStageCodegen (6) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_store_sk,ss_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] + CometProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] + CometBroadcastHashJoin [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand,ss_store_sk,ave] + CometProject [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometBroadcastHashJoin [s_store_name,ss_store_sk,ss_item_sk,revenue,i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] + CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] + CometFilter [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 + CometFilter [ss_store_sk,ss_item_sk,revenue] + CometHashAggregate [sum] [ss_store_sk,ss_item_sk,revenue,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_store_sk,ss_item_sk] #2 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #4 + CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometBroadcastExchange [ss_store_sk,ave] #5 + CometFilter [ss_store_sk,ave] + CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] + CometExchange [ss_store_sk] #6 + CometHashAggregate [revenue] [ss_store_sk,sum,count] + CometHashAggregate [ss_item_sk,sum] [ss_store_sk,revenue,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_store_sk,ss_item_sk] #7 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/explain.txt index 284cecf7d9..35c81a69d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/explain.txt @@ -1,320 +1,287 @@ == Physical Plan == -TakeOrderedAndProject (55) -+- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- Union (51) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.time_dim (16) - : +- BroadcastExchange (27) - : +- * Project (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.ship_mode (23) - +- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- ReusedExchange (45) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (51) ++- CometTakeOrderedAndProject (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometUnion (46) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.ship_mode (20) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Condition : isnotnull(w_warehouse_sk#8) -(7) BroadcastExchange +(5) CometProject Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#13, 2, true, false, true) AS w_state#15, w_country#14] + +(6) CometBroadcastExchange +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_warehouse_sk#3] -Right keys [1]: [w_warehouse_sk#8] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight -(9) Project [codegen id : 5] -Output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +(8) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_year#16, d_moy#17] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] +(10) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] -(13) BroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_sold_date_sk#7] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(13) CometProject +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] -(15) Project [codegen id : 5] -Output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] - -(16) Scan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#18, t_time#19] +(14) CometNativeScan parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#19, t_time#20] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [t_time_sk#18, t_time#19] - -(18) Filter [codegen id : 3] -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) +(15) CometFilter +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) -(19) Project [codegen id : 3] -Output [1]: [t_time_sk#18] -Input [2]: [t_time_sk#18, t_time#19] +(16) CometProject +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] -(20) BroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_sold_time_sk#1] -Right keys [1]: [t_time_sk#18] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight -(22) Project [codegen id : 5] -Output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] +(19) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] -(23) Scan parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] +(20) CometNativeScan parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] -PushedFilters: [In(sm_carrier, [BARIAN ,DHL ]), IsNotNull(sm_ship_mode_sk)] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] - -(25) Filter [codegen id : 4] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) - -(26) Project [codegen id : 4] -Output [1]: [sm_ship_mode_sk#20] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] - -(27) BroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_mode_sk#2] -Right keys [1]: [sm_ship_mode_sk#20] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] - -(30) HashAggregate [codegen id : 5] -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Results [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] - -(31) Exchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141] -Results [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, DHL,BARIAN AS ship_carriers#142, d_year#16 AS year#143, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118 AS jan_sales#144, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119 AS feb_sales#145, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120 AS mar_sales#146, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121 AS apr_sales#147, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122 AS may_sales#148, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123 AS jun_sales#149, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124 AS jul_sales#150, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125 AS aug_sales#151, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126 AS sep_sales#152, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127 AS oct_sales#153, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128 AS nov_sales#154, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129 AS dec_sales#155, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130 AS jan_net#156, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131 AS feb_net#157, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132 AS mar_net#158, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133 AS apr_net#159, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134 AS may_net#160, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135 AS jun_net#161, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136 AS jul_net#162, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137 AS aug_net#163, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138 AS sep_net#164, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139 AS oct_net#165, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140 AS nov_net#166, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141 AS dec_net#167] - -(33) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] +(21) CometFilter +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#22, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) + +(22) CometProject +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] + +(23) CometBroadcastExchange +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] + +(24) CometBroadcastHashJoin +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(26) CometHashAggregate +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(27) CometExchange +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometHashAggregate +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(29) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#174)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] - -(35) Filter [codegen id : 11] -Input [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] -Condition : ((isnotnull(cs_warehouse_sk#170) AND isnotnull(cs_sold_time_sk#168)) AND isnotnull(cs_ship_mode_sk#169)) - -(36) ReusedExchange [Reuses operator id: 7] -Output [7]: [w_warehouse_sk#175, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_warehouse_sk#170] -Right keys [1]: [w_warehouse_sk#175] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [12]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] -Input [14]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_sk#175, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] - -(39) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#182, d_year#183, d_moy#184] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#174] -Right keys [1]: [d_date_sk#182] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [13]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [15]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_date_sk#182, d_year#183, d_moy#184] - -(42) ReusedExchange [Reuses operator id: 20] -Output [1]: [t_time_sk#185] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_time_sk#168] -Right keys [1]: [t_time_sk#185] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [12]: [cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [14]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184, t_time_sk#185] - -(45) ReusedExchange [Reuses operator id: 27] -Output [1]: [sm_ship_mode_sk#186] - -(46) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_mode_sk#169] -Right keys [1]: [sm_ship_mode_sk#186] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 11] -Output [11]: [cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [13]: [cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184, sm_ship_mode_sk#186] - -(48) HashAggregate [codegen id : 11] -Input [11]: [cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Keys [7]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183] -Functions [24]: [partial_sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234] -Results [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] - -(49) Exchange -Input [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] -Arguments: hashpartitioning(w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(50) HashAggregate [codegen id : 12] -Input [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] -Keys [7]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183] -Functions [24]: [sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#283, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#284, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#285, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#286, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#287, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#288, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#289, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#290, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#291, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#292, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#293, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#294, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#295, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#296, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#297, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#298, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#299, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#300, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#301, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#302, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#303, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#304, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#305, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#306] -Results [32]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, DHL,BARIAN AS ship_carriers#307, d_year#183 AS year#308, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#283 AS jan_sales#309, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#284 AS feb_sales#310, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#285 AS mar_sales#311, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#286 AS apr_sales#312, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#287 AS may_sales#313, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#288 AS jun_sales#314, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#289 AS jul_sales#315, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#290 AS aug_sales#316, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#291 AS sep_sales#317, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#292 AS oct_sales#318, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#293 AS nov_sales#319, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#294 AS dec_sales#320, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#295 AS jan_net#321, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#296 AS feb_net#322, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#297 AS mar_net#323, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#298 AS apr_net#324, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#299 AS may_net#325, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#300 AS jun_net#326, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#301 AS jul_net#327, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#302 AS aug_net#328, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#303 AS sep_net#329, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#304 AS oct_net#330, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#305 AS nov_net#331, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#306 AS dec_net#332] - -(51) Union - -(52) HashAggregate [codegen id : 13] -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#144, feb_sales#145, mar_sales#146, apr_sales#147, may_sales#148, jun_sales#149, jul_sales#150, aug_sales#151, sep_sales#152, oct_sales#153, nov_sales#154, dec_sales#155, jan_net#156, feb_net#157, mar_net#158, apr_net#159, may_net#160, jun_net#161, jul_net#162, aug_net#163, sep_net#164, oct_net#165, nov_net#166, dec_net#167] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [partial_sum(jan_sales#144), partial_sum(feb_sales#145), partial_sum(mar_sales#146), partial_sum(apr_sales#147), partial_sum(may_sales#148), partial_sum(jun_sales#149), partial_sum(jul_sales#150), partial_sum(aug_sales#151), partial_sum(sep_sales#152), partial_sum(oct_sales#153), partial_sum(nov_sales#154), partial_sum(dec_sales#155), partial_sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#156), partial_sum(feb_net#157), partial_sum(mar_net#158), partial_sum(apr_net#159), partial_sum(may_net#160), partial_sum(jun_net#161), partial_sum(jul_net#162), partial_sum(aug_net#163), partial_sum(sep_net#164), partial_sum(oct_net#165), partial_sum(nov_net#166), partial_sum(dec_net#167)] -Aggregate Attributes [72]: [sum#333, isEmpty#334, sum#335, isEmpty#336, sum#337, isEmpty#338, sum#339, isEmpty#340, sum#341, isEmpty#342, sum#343, isEmpty#344, sum#345, isEmpty#346, sum#347, isEmpty#348, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404] -Results [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] - -(53) Exchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 14] -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [sum(jan_sales#144), sum(feb_sales#145), sum(mar_sales#146), sum(apr_sales#147), sum(may_sales#148), sum(jun_sales#149), sum(jul_sales#150), sum(aug_sales#151), sum(sep_sales#152), sum(oct_sales#153), sum(nov_sales#154), sum(dec_sales#155), sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#156), sum(feb_net#157), sum(mar_net#158), sum(apr_net#159), sum(may_net#160), sum(jun_net#161), sum(jul_net#162), sum(aug_net#163), sum(sep_net#164), sum(oct_net#165), sum(nov_net#166), sum(dec_net#167)] -Aggregate Attributes [36]: [sum(jan_sales#144)#477, sum(feb_sales#145)#478, sum(mar_sales#146)#479, sum(apr_sales#147)#480, sum(may_sales#148)#481, sum(jun_sales#149)#482, sum(jul_sales#150)#483, sum(aug_sales#151)#484, sum(sep_sales#152)#485, sum(oct_sales#153)#486, sum(nov_sales#154)#487, sum(dec_sales#155)#488, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#489, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#490, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#491, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#492, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#493, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#494, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#495, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#496, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#497, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#498, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#499, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#500, sum(jan_net#156)#501, sum(feb_net#157)#502, sum(mar_net#158)#503, sum(apr_net#159)#504, sum(may_net#160)#505, sum(jun_net#161)#506, sum(jul_net#162)#507, sum(aug_net#163)#508, sum(sep_net#164)#509, sum(oct_net#165)#510, sum(nov_net#166)#511, sum(dec_net#167)#512] -Results [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum(jan_sales#144)#477 AS jan_sales#513, sum(feb_sales#145)#478 AS feb_sales#514, sum(mar_sales#146)#479 AS mar_sales#515, sum(apr_sales#147)#480 AS apr_sales#516, sum(may_sales#148)#481 AS may_sales#517, sum(jun_sales#149)#482 AS jun_sales#518, sum(jul_sales#150)#483 AS jul_sales#519, sum(aug_sales#151)#484 AS aug_sales#520, sum(sep_sales#152)#485 AS sep_sales#521, sum(oct_sales#153)#486 AS oct_sales#522, sum(nov_sales#154)#487 AS nov_sales#523, sum(dec_sales#155)#488 AS dec_sales#524, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#489 AS jan_sales_per_sq_foot#525, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#490 AS feb_sales_per_sq_foot#526, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#491 AS mar_sales_per_sq_foot#527, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#492 AS apr_sales_per_sq_foot#528, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#493 AS may_sales_per_sq_foot#529, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#494 AS jun_sales_per_sq_foot#530, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#495 AS jul_sales_per_sq_foot#531, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#496 AS aug_sales_per_sq_foot#532, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#497 AS sep_sales_per_sq_foot#533, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#498 AS oct_sales_per_sq_foot#534, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#499 AS nov_sales_per_sq_foot#535, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#500 AS dec_sales_per_sq_foot#536, sum(jan_net#156)#501 AS jan_net#537, sum(feb_net#157)#502 AS feb_net#538, sum(mar_net#158)#503 AS mar_net#539, sum(apr_net#159)#504 AS apr_net#540, sum(may_net#160)#505 AS may_net#541, sum(jun_net#161)#506 AS jun_net#542, sum(jul_net#162)#507 AS jul_net#543, sum(aug_net#163)#508 AS aug_net#544, sum(sep_net#164)#509 AS sep_net#545, sum(oct_net#165)#510 AS oct_net#546, sum(nov_net#166)#511 AS nov_net#547, sum(dec_net#167)#512 AS dec_net#548] - -(55) TakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#513, feb_sales#514, mar_sales#515, apr_sales#516, may_sales#517, jun_sales#518, jul_sales#519, aug_sales#520, sep_sales#521, oct_sales#522, nov_sales#523, dec_sales#524, jan_sales_per_sq_foot#525, feb_sales_per_sq_foot#526, mar_sales_per_sq_foot#527, apr_sales_per_sq_foot#528, may_sales_per_sq_foot#529, jun_sales_per_sq_foot#530, jul_sales_per_sq_foot#531, aug_sales_per_sq_foot#532, sep_sales_per_sq_foot#533, oct_sales_per_sq_foot#534, nov_sales_per_sq_foot#535, dec_sales_per_sq_foot#536, jan_net#537, feb_net#538, mar_net#539, apr_net#540, may_net#541, jun_net#542, jul_net#543, aug_net#544, sep_net#545, oct_net#546, nov_net#547, dec_net#548] -Arguments: 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#513, feb_sales#514, mar_sales#515, apr_sales#516, may_sales#517, jun_sales#518, jul_sales#519, aug_sales#520, sep_sales#521, oct_sales#522, nov_sales#523, dec_sales#524, jan_sales_per_sq_foot#525, feb_sales_per_sq_foot#526, mar_sales_per_sq_foot#527, apr_sales_per_sq_foot#528, may_sales_per_sq_foot#529, jun_sales_per_sq_foot#530, jul_sales_per_sq_foot#531, aug_sales_per_sq_foot#532, sep_sales_per_sq_foot#533, oct_sales_per_sq_foot#534, nov_sales_per_sq_foot#535, dec_sales_per_sq_foot#536, jan_net#537, feb_net#538, mar_net#539, apr_net#540, may_net#541, jun_net#542, jul_net#543, aug_net#544, sep_net#545, oct_net#546, nov_net#547, dec_net#548] +(30) CometFilter +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(32) CometBroadcastHashJoin +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight + +(33) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(34) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] + +(35) CometBroadcastHashJoin +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight + +(36) CometProject +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(37) ReusedExchange [Reuses operator id: 17] +Output [1]: [t_time_sk#88] + +(38) CometBroadcastHashJoin +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight + +(39) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(40) ReusedExchange [Reuses operator id: 23] +Output [1]: [sm_ship_mode_sk#89] + +(41) CometBroadcastHashJoin +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight + +(42) CometProject +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(43) CometHashAggregate +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(44) CometExchange +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(45) CometHashAggregate +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(46) CometUnion +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] + +(47) CometHashAggregate +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] + +(48) CometExchange +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(49) CometHashAggregate +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] + +(50) CometTakeOrderedAndProject +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] + +(51) CometColumnarToRow [codegen id : 1] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/extended.txt new file mode 100644 index 0000000000..3dee6b4b4e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/extended.txt @@ -0,0 +1,64 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.time_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.ship_mode + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.ship_mode + +Comet accelerated 61 out of 61 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/simplified.txt index 9a2e8adb87..64ef752d91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/simplified.txt @@ -1,83 +1,53 @@ -TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - WholeStageCodegen (14) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - WholeStageCodegen (13) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - WholeStageCodegen (5) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk] - Project [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_sold_time_sk,ws_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [t_time_sk] - Filter [t_time,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_time] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [sm_ship_mode_sk] - Filter [sm_carrier,sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] - WholeStageCodegen (12) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 - WholeStageCodegen (11) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk] - Project [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_sold_time_sk,t_time_sk] - Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_sold_time_sk,cs_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_moy] #4 - InputAdapter - ReusedExchange [t_time_sk] #5 - InputAdapter - ReusedExchange [sm_ship_mode_sk] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 + CometHashAggregate [jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 + CometHashAggregate [d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [t_time_sk] #5 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_time] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_time] + CometBroadcastExchange [sm_ship_mode_sk] #6 + CometProject [sm_ship_mode_sk] + CometFilter [sm_ship_mode_sk,sm_carrier] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 + CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/explain.txt index 6ceedf2f92..e77331bcaf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/explain.txt @@ -1,179 +1,170 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Expand (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.item (17) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (30) ++- * Filter (29) + +- Window (28) + +- * CometColumnarToRow (27) + +- CometSort (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.item (15) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +(5) CometProject Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +(8) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(14) BroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] -(16) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(17) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) -(19) Filter [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(20) BroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(22) Project [codegen id : 4] -Output [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] -(23) Expand [codegen id : 4] -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(21) CometExpand +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] -(24) HashAggregate [codegen id : 4] -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(22) CometHashAggregate +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -(25) Exchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(24) CometHashAggregate +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31] -Results [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31 AS sumsales#32] -(27) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometExchange +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(26) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] -(28) Sort [codegen id : 6] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 +(27) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [rank(sumsales#32) windowspecdefinition(i_category#18, sumsales#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [i_category#18], [sumsales#32 DESC NULLS LAST] +(28) Window +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] -(30) Filter [codegen id : 7] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Condition : (rk#33 <= 100) +(29) Filter [codegen id : 2] +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#32 ASC NULLS FIRST, rk#33 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] +(30) TakeOrderedAndProject +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/extended.txt new file mode 100644 index 0000000000..255bfe16ac --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/extended.txt @@ -0,0 +1,32 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 26 out of 29 eligible operators (89%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/simplified.txt index 4205eed4b0..a14826897e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/simplified.txt @@ -1,48 +1,34 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [rk] InputAdapter Window [sumsales,i_category] - WholeStageCodegen (6) - Sort [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - Expand [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - Project [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometExchange [i_category] #1 + CometHashAggregate [spark_grouping_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/explain.txt index 6acd6c0e99..ca5c9ff3ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/explain.txt @@ -1,248 +1,223 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Project (24) - : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.store (11) - : : : +- BroadcastExchange (22) - : : : +- * Project (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.household_demographics (18) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.customer_address (25) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.customer (34) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometHashAggregate (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometNativeScan parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_dom#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_dom#12] Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(5) CometProject Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Arguments: [d_date_sk#10], [d_date_sk#10] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#10] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#9] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(10) Project [codegen id : 5] -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +(8) CometProject Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#13, s_city#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#13, s_city#14] Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#13] +(11) CometProject Input [2]: [s_store_sk#13, s_city#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +(14) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#15] +(17) CometProject Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#15] -(23) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight -(24) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +(20) CometProject Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(25) Scan parquet spark_catalog.default.customer_address +(21) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_city#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#18, ca_city#19] - -(27) Filter [codegen id : 4] +(22) CometFilter Input [2]: [ca_address_sk#18, ca_city#19] Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) -(28) BroadcastExchange +(23) CometBroadcastExchange Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [ca_address_sk#18, ca_city#19] -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#18] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight -(30) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] +(25) CometProject Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] -(31) HashAggregate [codegen id : 5] +(26) CometHashAggregate Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum#20, sum#21, sum#22] -Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -(32) Exchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 8] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] +(28) CometHashAggregate +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#26, sum(UnscaledValue(ss_ext_list_price#7))#27, sum(UnscaledValue(ss_ext_tax#8))#28] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#26,17,2) AS extended_price#30, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#27,17,2) AS list_price#31, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#28,17,2) AS extended_tax#32] -(34) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(29) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(30) CometFilter +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Condition : (isnotnull(c_customer_sk#23) AND isnotnull(c_current_addr_sk#24)) + +(31) CometProject +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#23, c_current_addr_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#25, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#26, 30, true, false, true) AS c_last_name#28] -(36) Filter [codegen id : 6] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#34)) +(32) CometBroadcastExchange +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] -(37) BroadcastExchange -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(33) CometBroadcastHashJoin +Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32] +Right output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_customer_sk#1], [c_customer_sk#23], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#33] -Join type: Inner -Join condition: None +(34) CometProject +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] -(39) Project [codegen id : 8] -Output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(35) ReusedExchange [Reuses operator id: 23] +Output [2]: [ca_address_sk#33, ca_city#34] -(40) ReusedExchange [Reuses operator id: 28] -Output [2]: [ca_address_sk#37, ca_city#38] +(36) CometBroadcastHashJoin +Left output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Right output [2]: [ca_address_sk#33, ca_city#34] +Arguments: [c_current_addr_sk#24], [ca_address_sk#33], Inner, NOT (ca_city#34 = bought_city#29), BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#37] -Join type: Inner -Join condition: NOT (ca_city#38 = bought_city#29) +(37) CometProject +Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28, ca_address_sk#33, ca_city#34] +Arguments: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -(42) Project [codegen id : 8] -Output [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36, ca_address_sk#37, ca_city#38] +(38) CometTakeOrderedAndProject +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#28 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#28,c_first_name#27,ca_city#34,bought_city#29,ss_ticket_number#5,extended_price#30,extended_tax#32,list_price#31]), [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], 100, 0, [c_last_name#28 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -(43) TakeOrderedAndProject -Input [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Arguments: 100, [c_last_name#36 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +(39) CometColumnarToRow [codegen id : 1] +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/extended.txt new file mode 100644 index 0000000000..2cc4ab6ef2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 40 out of 40 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/simplified.txt index 84a31f6a2b..fe430a0ce4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/simplified.txt @@ -1,63 +1,41 @@ -TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price] - WholeStageCodegen (8) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (5) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ca_address_sk,ca_city] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] + CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] + CometBroadcastHashJoin [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] + CometProject [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometHashAggregate [ss_addr_sk,sum,sum,sum] [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,ca_city,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax))] + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/explain.txt index fe47588c7c..3736f4ebe2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/explain.txt @@ -1,267 +1,264 @@ == Physical Plan == TakeOrderedAndProject (46) +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin LeftAnti BuildRight (28) - : : :- * BroadcastHashJoin LeftAnti BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.customer_address (30) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer + +- * CometColumnarToRow (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometNativeScan parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (39) + +- * CometColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (35) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(6) CometProject Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] +(9) CometProject Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#4] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] + +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#9] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#11, cs_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#14] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#11, cs_sold_date_sk#12] +Right output [1]: [d_date_sk#6] +Arguments: [cs_sold_date_sk#12], [d_date_sk#6], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#11, cs_sold_date_sk#12, d_date_sk#6] +Arguments: [cs_ship_customer_sk#11], [cs_ship_customer_sk#11] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#12] -Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#11] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#12] +Right keys [1]: [cs_ship_customer_sk#11] Join type: LeftAnti Join condition: None -(29) Project [codegen id : 9] +(27) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(30) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#15, ca_state#16] +(28) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [GA,KY,NM]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#15, ca_state#16] +(29) CometFilter +Input [2]: [ca_address_sk#13, ca_state#14] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#13)) -(32) Filter [codegen id : 7] -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : (ca_state#16 IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) +(30) CometProject +Input [2]: [ca_address_sk#13, ca_state#14] +Arguments: [ca_address_sk#13], [ca_address_sk#13] -(33) Project [codegen id : 7] -Output [1]: [ca_address_sk#15] -Input [2]: [ca_address_sk#15, ca_state#16] +(31) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#13] -(34) BroadcastExchange -Input [1]: [ca_address_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(32) BroadcastExchange +Input [1]: [ca_address_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 9] +(33) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#15] +Right keys [1]: [ca_address_sk#13] Join type: Inner Join condition: None -(36) Project [codegen id : 9] +(34) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#13] -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(35) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(36) CometFilter +Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +Condition : isnotnull(cd_demo_sk#15) -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Condition : isnotnull(cd_demo_sk#17) +(37) CometProject +Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +Arguments: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24], [cd_demo_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#16, 1, true, false, true) AS cd_gender#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#17, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#18, 20, true, false, true) AS cd_education_status#23, cd_purchase_estimate#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#20, 10, true, false, true) AS cd_credit_rating#24] -(40) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(38) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] -(41) BroadcastHashJoin [codegen id : 9] +(39) BroadcastExchange +Input [6]: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] +Right keys [1]: [cd_demo_sk#15] Join type: Inner Join condition: None -(42) Project [codegen id : 9] -Output [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(41) Project [codegen id : 5] +Output [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] -(43) HashAggregate [codegen id : 9] -Input [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Keys [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(42) HashAggregate [codegen id : 5] +Input [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] +Aggregate Attributes [1]: [count#25] +Results [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] + +(43) CometColumnarExchange +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] +Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(44) Exchange -Input [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] -Arguments: hashpartitioning(cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(44) CometColumnarToRow [codegen id : 6] +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] -(45) HashAggregate [codegen id : 10] -Input [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] -Keys [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(45) HashAggregate [codegen id : 6] +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] +Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, count(1)#25 AS cnt1#26, cd_purchase_estimate#21, count(1)#25 AS cnt2#27, cd_credit_rating#22, count(1)#25 AS cnt3#28] +Aggregate Attributes [1]: [count(1)#27] +Results [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, count(1)#27 AS cnt1#28, cd_purchase_estimate#19, count(1)#27 AS cnt2#29, cd_credit_rating#24, count(1)#27 AS cnt3#30] (46) TakeOrderedAndProject -Input [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#26, cd_purchase_estimate#21, cnt2#27, cd_credit_rating#22, cnt3#28] -Arguments: 100, [cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_education_status#20 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#22 ASC NULLS FIRST], [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#26, cd_purchase_estimate#21, cnt2#27, cd_credit_rating#22, cnt3#28] +Input [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#28, cd_purchase_estimate#19, cnt2#29, cd_credit_rating#24, cnt3#30] +Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FIRST, cd_education_status#23 ASC NULLS FIRST, cd_purchase_estimate#19 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST], [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#28, cd_purchase_estimate#19, cnt2#29, cd_credit_rating#24, cnt3#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/extended.txt new file mode 100644 index 0000000000..0fa566e7ff --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/extended.txt @@ -0,0 +1,54 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 46 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/simplified.txt index 3ca9dff655..69d8e49a63 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/simplified.txt @@ -1,70 +1,62 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/explain.txt index 5c2f1ed7e5..5f35ffe250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/explain.txt @@ -1,198 +1,179 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.item (18) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.promotion (24) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.promotion (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(8) CometProject Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +(14) CometProject Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(18) Scan parquet spark_catalog.default.item +(15) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(21) BroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] -(24) Scan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(21) CometNativeScan parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) -(26) Filter [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] -(27) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] -(28) BroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#17] -Join type: Inner -Join condition: None +(26) CometProject +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] -(30) Project [codegen id : 5] -Output [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] - -(31) HashAggregate [codegen id : 5] -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(27) CometHashAggregate +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -(32) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 6] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#36, avg(UnscaledValue(ss_list_price#5))#37, avg(UnscaledValue(ss_coupon_amt#7))#38, avg(UnscaledValue(ss_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(ss_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(ss_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(ss_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(ss_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] -(34) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/extended.txt new file mode 100644 index 0000000000..4848e7820f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/simplified.txt index 8410ad41e8..0867ca71bb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/simplified.txt @@ -1,50 +1,33 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] + CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/explain.txt index 1cb84c5726..c46734ce5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/explain.txt @@ -1,266 +1,269 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * Project (45) - +- Window (44) - +- * Sort (43) - +- Exchange (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Expand (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- BroadcastExchange (35) - +- * BroadcastHashJoin LeftSemi BuildRight (34) - :- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (33) - +- * Project (32) - +- * Filter (31) - +- Window (30) - +- * Sort (29) - +- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Filter (16) - : : +- * ColumnarToRow (15) - : : +- Scan parquet spark_catalog.default.store_sales (14) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - +- ReusedExchange (23) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (48) ++- * Project (47) + +- Window (46) + +- * CometColumnarToRow (45) + +- CometSort (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Expand (38) + +- * Project (37) + +- * BroadcastHashJoin Inner BuildRight (36) + :- * CometColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (35) + +- * Project (34) + +- * BroadcastHashJoin LeftSemi BuildRight (33) + :- * CometColumnarToRow (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.store (10) + +- BroadcastExchange (32) + +- * Project (31) + +- * Filter (30) + +- Window (29) + +- * CometColumnarToRow (28) + +- CometSort (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometFilter (14) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + +- ReusedExchange (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] +(8) CometProject Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] + +(9) CometColumnarToRow [codegen id : 4] +Input [2]: [ss_store_sk#1, ss_net_profit#2] -(11) Scan parquet spark_catalog.default.store +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 7] +(11) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) -(13) Filter [codegen id : 7] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) -(14) Scan parquet spark_catalog.default.store_sales +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(16) Filter [codegen id : 4] +(14) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_state#13] +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#6, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] +(16) CometFilter +Input [2]: [s_store_sk#6, s_state#8] +Condition : isnotnull(s_store_sk#6) -(19) Filter [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] -Condition : isnotnull(s_store_sk#12) +(17) CometProject +Input [2]: [s_store_sk#6, s_state#8] +Arguments: [s_store_sk#6, s_state#12], [s_store_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#12] -(20) BroadcastExchange -Input [2]: [s_store_sk#12, s_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(18) CometBroadcastExchange +Input [2]: [s_store_sk#6, s_state#12] +Arguments: [s_store_sk#6, s_state#12] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#9] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#6, s_state#12] +Arguments: [ss_store_sk#9], [s_store_sk#6], Inner, BuildRight -(22) Project [codegen id : 4] -Output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#6, s_state#12] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12], [ss_net_profit#10, ss_sold_date_sk#11, s_state#12] -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#14] +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#4] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#11] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#11], [d_date_sk#4], Inner, BuildRight -(25) Project [codegen id : 4] -Output [2]: [ss_net_profit#10, s_state#13] -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12, d_date_sk#4] +Arguments: [ss_net_profit#10, s_state#12], [ss_net_profit#10, s_state#12] -(26) HashAggregate [codegen id : 4] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#12] +Keys [1]: [s_state#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] -(27) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(25) CometExchange +Input [2]: [s_state#12, sum#13] +Arguments: hashpartitioning(s_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#12, sum#13] +Keys [1]: [s_state#12] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] -(29) Sort [codegen id : 5] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 +(27) CometSort +Input [3]: [s_state#12, _w0#14, s_state#12] +Arguments: [s_state#12, _w0#14, s_state#12], [s_state#12 ASC NULLS FIRST, _w0#14 DESC NULLS LAST] -(30) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#12, _w0#14, s_state#12] -(31) Filter [codegen id : 6] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] -Condition : (ranking#19 <= 5) +(29) Window +Input [3]: [s_state#12, _w0#14, s_state#12] +Arguments: [rank(_w0#14) windowspecdefinition(s_state#12, _w0#14 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#15], [s_state#12], [_w0#14 DESC NULLS LAST] -(32) Project [codegen id : 6] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] +(30) Filter [codegen id : 2] +Input [4]: [s_state#12, _w0#14, s_state#12, ranking#15] +Condition : (ranking#15 <= 5) -(33) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] +(31) Project [codegen id : 2] +Output [1]: [s_state#12] +Input [4]: [s_state#12, _w0#14, s_state#12, ranking#15] -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(32) BroadcastExchange +Input [1]: [s_state#12] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] + +(33) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#12] Join type: LeftSemi Join condition: None -(35) BroadcastExchange +(34) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#16] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(36) BroadcastHashJoin [codegen id : 8] +(35) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(36) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(37) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(37) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_state#16, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#16] -(38) Expand [codegen id : 8] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] +(38) Expand [codegen id : 4] +Input [3]: [ss_net_profit#2, s_state#16, s_county#7] +Arguments: [[ss_net_profit#2, s_state#16, s_county#7, 0], [ss_net_profit#2, s_state#16, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#17, s_county#18, spark_grouping_id#19] -(39) HashAggregate [codegen id : 8] -Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] +(39) HashAggregate [codegen id : 4] +Input [4]: [ss_net_profit#2, s_state#17, s_county#18, spark_grouping_id#19] +Keys [3]: [s_state#17, s_county#18, spark_grouping_id#19] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#23] -Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Aggregate Attributes [1]: [sum#20] +Results [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] + +(40) CometColumnarExchange +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] +Arguments: hashpartitioning(s_state#17, s_county#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) Exchange -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(41) CometColumnarToRow [codegen id : 5] +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] -(41) HashAggregate [codegen id : 9] -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] +(42) HashAggregate [codegen id : 5] +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] +Keys [3]: [s_state#17, s_county#18, spark_grouping_id#19] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#23, s_state#17, s_county#18, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS lochierarchy#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS _w0#25, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS _w1#26, CASE WHEN (cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint) = 0) THEN s_state#17 END AS _w2#27] + +(43) CometColumnarExchange +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: hashpartitioning(_w1#26, _w2#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(42) Exchange -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(44) CometSort +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27], [_w1#26 ASC NULLS FIRST, _w2#27 ASC NULLS FIRST, _w0#25 DESC NULLS LAST] -(43) Sort [codegen id : 10] -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST], false, 0 +(45) CometColumnarToRow [codegen id : 6] +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] -(44) Window -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] +(46) Window +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: [rank(_w0#25) windowspecdefinition(_w1#26, _w2#27, _w0#25 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#28], [_w1#26, _w2#27], [_w0#25 DESC NULLS LAST] -(45) Project [codegen id : 11] -Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] +(47) Project [codegen id : 7] +Output [5]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] +Input [8]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27, rank_within_parent#28] -(46) TakeOrderedAndProject -Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +(48) TakeOrderedAndProject +Input [5]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] +Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN s_state#17 END ASC NULLS FIRST, rank_within_parent#28 ASC NULLS FIRST], [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/extended.txt new file mode 100644 index 0000000000..8c887178fe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/extended.txt @@ -0,0 +1,53 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Filter + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 31 out of 46 eligible operators (67%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/simplified.txt index a98d640eee..36e1ba57b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/simplified.txt @@ -1,71 +1,64 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (11) + WholeStageCodegen (7) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (10) - Sort [_w1,_w2,_w0] + WholeStageCodegen (6) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (9) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (8) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WholeStageCodegen (5) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] - InputAdapter - Exchange [s_state] #6 - WholeStageCodegen (4) - HashAggregate [s_state,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_state] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_net_profit,ss_sold_date_sk,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - ReusedExchange [d_date_sk] #3 + CometFilter [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #6 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #7 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/explain.txt index 1d8a3199e5..b0d17c994e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/explain.txt @@ -1,240 +1,214 @@ == Physical Plan == -* Sort (42) -+- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildLeft (29) - : :- BroadcastExchange (5) - : : +- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.item (1) - : +- Union (28) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.web_sales (6) - : : +- BroadcastExchange (13) - : : +- * Project (12) - : : +- * Filter (11) - : : +- * ColumnarToRow (10) - : : +- Scan parquet spark_catalog.default.date_dim (9) - : :- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) - : : : +- Scan parquet spark_catalog.default.catalog_sales (16) - : : +- ReusedExchange (19) - : +- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.store_sales (22) - : +- ReusedExchange (25) - +- BroadcastExchange (35) - +- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) - +- Scan parquet spark_catalog.default.time_dim (31) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (37) ++- CometSort (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometBroadcastExchange (4) + : : +- CometProject (3) + : : +- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.item (1) + : +- CometUnion (23) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometFilter (6) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (5) + : : +- CometBroadcastExchange (10) + : : +- CometProject (9) + : : +- CometFilter (8) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (7) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (13) + : : +- ReusedExchange (15) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (19) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (18) + : +- ReusedExchange (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometNativeScan parquet spark_catalog.default.time_dim (26) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 1] -Output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] +(3) CometProject Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#3, 50, true, false, true) AS i_brand#5] -(5) BroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(4) CometBroadcastExchange +Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(6) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +(5) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#8)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +(6) CometFilter +Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) -(8) Filter [codegen id : 3] -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) - -(9) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(7) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(10) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(11) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +(8) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) -(12) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] -(13) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(10) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(15) Project [codegen id : 3] -Output [3]: [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] +(12) CometProject +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] -(16) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(13) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 5] -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(14) CometFilter +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) -(18) Filter [codegen id : 5] -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) +(15) ReusedExchange [Reuses operator id: 10] +Output [1]: [d_date_sk#20] -(19) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#19] +(16) CometBroadcastHashJoin +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#18] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(17) CometProject +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] -(21) Project [codegen id : 5] -Output [3]: [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] - -(22) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(18) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 7] -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] - -(24) Filter [codegen id : 7] -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) +(19) CometFilter +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) -(25) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#27] +(20) ReusedExchange [Reuses operator id: 10] +Output [1]: [d_date_sk#28] -(26) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight -(27) Project [codegen id : 7] -Output [3]: [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] +(22) CometProject +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] -(28) Union +(23) CometUnion +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [sold_item_sk#13] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft -(30) Project [codegen id : 9] -Output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] +(25) CometProject +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] -(31) Scan parquet spark_catalog.default.time_dim -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] +(26) CometNativeScan parquet spark_catalog.default.time_dim +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [Or(EqualTo(t_meal_time,breakfast ),EqualTo(t_meal_time,dinner )), IsNotNull(t_time_sk)] +PushedFilters: [IsNotNull(t_time_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 8] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(33) Filter [codegen id : 8] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) - -(34) Project [codegen id : 8] -Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(35) BroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(36) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [time_sk#14] -Right keys [1]: [t_time_sk#31] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 9] -Output [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] - -(38) HashAggregate [codegen id : 9] -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum#35] -Results [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] - -(39) Exchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 10] -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#12))#37] -Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#3 AS brand#39, t_hour#32, t_minute#33, MakeDecimal(sum(UnscaledValue(ext_price#12))#37,17,2) AS ext_price#40] - -(41) Exchange -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: rangepartitioning(ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(42) Sort [codegen id : 11] -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST], true, 0 +(27) CometFilter +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#32)) + +(28) CometProject +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] + +(29) CometBroadcastExchange +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] + +(30) CometBroadcastHashJoin +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight + +(31) CometProject +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] + +(32) CometHashAggregate +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] + +(33) CometExchange +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(34) CometHashAggregate +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] + +(35) CometExchange +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: rangepartitioning(ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(36) CometSort +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39], [ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST] + +(37) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/extended.txt new file mode 100644 index 0000000000..4bd9ef86c7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometUnion + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.time_dim + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/simplified.txt index bdc48db383..be00094510 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/simplified.txt @@ -1,65 +1,39 @@ -WholeStageCodegen (11) - Sort [ext_price,brand_id] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [ext_price,brand_id] #1 - WholeStageCodegen (10) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,t_hour,t_minute] #2 - WholeStageCodegen (9) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum] - Project [i_brand_id,i_brand,ext_price,t_hour,t_minute] - BroadcastHashJoin [time_sk,t_time_sk] - Project [i_brand_id,i_brand,ext_price,time_sk] - BroadcastHashJoin [i_item_sk,sold_item_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] - InputAdapter - Union - WholeStageCodegen (3) - Project [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk,ws_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (5) - Project [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk,cs_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (7) - Project [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [t_time_sk,t_hour,t_minute] - Filter [t_meal_time,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] + CometSort [brand_id,brand,t_hour,t_minute,ext_price] + CometExchange [ext_price,brand_id] #1 + CometHashAggregate [sum] [brand_id,brand,t_hour,t_minute,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ext_price))] + CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 + CometHashAggregate [ext_price] [i_brand,i_brand_id,t_hour,t_minute,sum] + CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] + CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] + CometProject [i_brand_id,i_brand,ext_price,time_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometUnion [ext_price,sold_item_sk,time_sk] + CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 + CometProject [t_time_sk,t_hour,t_minute] + CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/explain.txt index e15d9df434..5ffc831b1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/explain.txt @@ -1,423 +1,369 @@ == Physical Plan == -TakeOrderedAndProject (74) -+- * HashAggregate (73) - +- Exchange (72) - +- * HashAggregate (71) - +- * Project (70) - +- * SortMergeJoin LeftOuter (69) - :- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * BroadcastHashJoin LeftOuter BuildRight (59) - : :- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (42) - : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : :- * Project (35) - : : : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : : : :- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Project (15) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : : : : :- * Project (9) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) - : : : : : : : : : +- Scan parquet spark_catalog.default.inventory (4) - : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) - : : : : : : : : +- Scan parquet spark_catalog.default.warehouse (10) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (22) - : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) - : : : : : +- Scan parquet spark_catalog.default.household_demographics (29) - : : : : +- BroadcastExchange (40) - : : : : +- * Project (39) - : : : : +- * Filter (38) - : : : : +- * ColumnarToRow (37) - : : : : +- Scan parquet spark_catalog.default.date_dim (36) - : : : +- BroadcastExchange (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) - : : : +- Scan parquet spark_catalog.default.date_dim (43) - : : +- BroadcastExchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.date_dim (49) - : +- BroadcastExchange (58) - : +- * Filter (57) - : +- * ColumnarToRow (56) - : +- Scan parquet spark_catalog.default.promotion (55) - +- * Sort (68) - +- Exchange (67) - +- * Project (66) - +- * Filter (65) - +- * ColumnarToRow (64) - +- Scan parquet spark_catalog.default.catalog_returns (63) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 10] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.inventory +(3) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -(8) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [inv_item_sk#9] -Join type: Inner -Join condition: (inv_quantity_on_hand#11 < cs_quantity#7) +(6) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight -(9) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +(7) CometProject Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] Condition : isnotnull(w_warehouse_sk#13) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] -(14) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [inv_warehouse_sk#10] -Right keys [1]: [w_warehouse_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight -(15) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +(12) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -(16) Scan parquet spark_catalog.default.item +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [2]: [i_item_sk#15, i_item_desc#16] Condition : isnotnull(i_item_sk#15) -(19) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [i_item_sk#15, i_item_desc#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_desc#16] -(20) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight -(21) Project [codegen id : 10] -Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(17) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(22) Scan parquet spark_catalog.default.customer_demographics +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,D), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(24) Filter [codegen id : 4] +(19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#17)) -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#17] +(20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] -(26) BroadcastExchange +(21) CometBroadcastExchange Input [1]: [cd_demo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [cd_demo_sk#17] -(27) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight -(28) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(23) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(29) Scan parquet spark_catalog.default.household_demographics +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(31) Filter [codegen id : 5] +(25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#19)) -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#19] +(26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] -(33) BroadcastExchange +(27) CometBroadcastExchange Input [1]: [hd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [hd_demo_sk#19] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight -(35) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(29) CometProject Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(36) Scan parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(38) Filter [codegen id : 6] +(31) CometFilter Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) -(39) Project [codegen id : 6] -Output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +(32) CometProject Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] -(40) BroadcastExchange +(33) CometBroadcastExchange Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight -(42) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(35) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(43) Scan parquet spark_catalog.default.date_dim +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] - -(45) Filter [codegen id : 7] +(37) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(46) BroadcastExchange +(38) CometBroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [plan_id=7] +Arguments: [d_date_sk#25, d_week_seq#26] -(47) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [d_week_seq#23, inv_date_sk#12] -Right keys [2]: [d_week_seq#26, d_date_sk#25] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight -(48) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(40) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(49) Scan parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] - -(51) Filter [codegen id : 8] +(42) CometFilter Input [2]: [d_date_sk#27, d_date#28] Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(52) BroadcastExchange +(43) CometBroadcastExchange Input [2]: [d_date_sk#27, d_date#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: [d_date_sk#27, d_date#28] -(53) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: (d_date#28 > date_add(d_date#22, 5)) +(44) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight -(54) Project [codegen id : 10] -Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(45) CometProject Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(55) Scan parquet spark_catalog.default.promotion +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(56) ColumnarToRow [codegen id : 9] -Input [1]: [p_promo_sk#29] - -(57) Filter [codegen id : 9] +(47) CometFilter Input [1]: [p_promo_sk#29] Condition : isnotnull(p_promo_sk#29) -(58) BroadcastExchange +(48) CometBroadcastExchange Input [1]: [p_promo_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [p_promo_sk#29] -(59) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_promo_sk#5] -Right keys [1]: [p_promo_sk#29] -Join type: LeftOuter -Join condition: None +(49) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight -(60) Project [codegen id : 10] -Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(50) CometProject Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(61) Exchange +(51) CometExchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(62) Sort [codegen id : 11] +(52) CometSort Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(63) Scan parquet spark_catalog.default.catalog_returns +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(64) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(65) Filter [codegen id : 12] +(54) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(66) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] +(55) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] -(67) Exchange +(56) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(68) Sort [codegen id : 13] +(57) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] -(69) SortMergeJoin [codegen id : 14] -Left keys [2]: [cs_item_sk#4, cs_order_number#6] -Right keys [2]: [cr_item_sk#30, cr_order_number#31] -Join type: LeftOuter -Join condition: None +(58) CometSortMergeJoin +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter -(70) Project [codegen id : 14] -Output [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(59) CometProject Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(71) HashAggregate [codegen id : 14] +(60) CometHashAggregate Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#33] -Results [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -(72) Exchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(61) CometExchange +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(73) HashAggregate [codegen id : 15] -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] +(62) CometHashAggregate +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#35] -Results [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count(1)#35 AS no_promo#36, count(1)#35 AS promo#37, count(1)#35 AS total_cnt#38] -(74) TakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] -Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] +(63) CometTakeOrderedAndProject +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + +(64) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/extended.txt new file mode 100644 index 0000000000..3bf43ef8c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/extended.txt @@ -0,0 +1,66 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_returns + +Comet accelerated 63 out of 63 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/simplified.txt index 37b3b330ae..073bd57a8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/simplified.txt @@ -1,114 +1,66 @@ -TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] - WholeStageCodegen (15) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] - InputAdapter - Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - WholeStageCodegen (14) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] - Project [w_warehouse_name,i_item_desc,d_week_seq] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (11) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #2 - WholeStageCodegen (10) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk] - Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - WholeStageCodegen (13) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] + CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] + CometProject [w_warehouse_name,i_item_desc,d_week_seq] + CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometExchange [cs_item_sk,cs_order_number] #2 + CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 + CometFilter [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 + CometFilter [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/explain.txt index 37ac7a21de..f4fcae1584 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/explain.txt @@ -1,208 +1,189 @@ == Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +(8) CometProject Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#9, s_county#10] Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] +(11) CometProject Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#9] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +(14) CometProject Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.0) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] +(17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#11] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] +(20) CometProject Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 1) AND (cnt#18 <= 5)) +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(25) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(cnt#18 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [cnt#18 DESC NULLS LAST], true, 0 +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/simplified.txt index 46f1e68f4a..145db18334 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/simplified.txt @@ -1,54 +1,35 @@ -WholeStageCodegen (7) - Sort [cnt] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [cnt] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [cnt] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/explain.txt index cd5821566e..571fdd617c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/explain.txt @@ -1,418 +1,383 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) -(6) Filter [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] -(7) BroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] - -(16) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(17) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] + +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(23) ReusedExchange [Reuses operator id: 7] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#15] -Right keys [1]: [ss_customer_sk#19] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(25) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] - -(32) HashAggregate [codegen id : 6] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(33) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(35) BroadcastExchange +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] + +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight + +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] + +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(37) Scan parquet spark_catalog.default.customer +(34) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(39) Filter [codegen id : 10] +(36) CometProject Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_customer_id#31)) +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(40) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] +(37) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(42) Filter [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_customer_sk#34) - -(43) BroadcastExchange -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [ws_bill_customer_sk#34] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36] -Input [7]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33, ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#37, d_year#38] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#36] -Right keys [1]: [d_date_sk#37] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Input [7]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36, d_date_sk#37, d_year#38] - -(49) HashAggregate [codegen id : 10] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum#39] -Results [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] - -(50) Exchange -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Arguments: hashpartitioning(c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#41] -Results [2]: [c_customer_id#31 AS customer_id#42, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#41,17,2) AS year_total#43] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#42, year_total#43] -Condition : (isnotnull(year_total#43) AND (year_total#43 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#42, year_total#43] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#42] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#42, year_total#43] - -(56) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) + +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] + +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight + +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] - -(58) Filter [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(c_customer_id#45)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [3]: [ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#44] -Right keys [1]: [ws_bill_customer_sk#48] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50] -Input [7]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#51, d_year#52] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Input [7]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(65) HashAggregate [codegen id : 14] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum#53] -Results [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] - -(66) Exchange -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#49))#41] -Results [2]: [c_customer_id#45 AS customer_id#55, MakeDecimal(sum(UnscaledValue(ws_net_paid#49))#41,17,2) AS year_total#56] - -(68) BroadcastExchange +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] + +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] + +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight + +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] + +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] + +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] + +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] + +(64) CometBroadcastExchange Input [2]: [customer_id#55, year_total#56] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] +Arguments: [customer_id#55, year_total#56] -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#55] -Join type: Inner -Join condition: (CASE WHEN (year_total#43 > 0.00) THEN (year_total#56 / year_total#43) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -(70) Project [codegen id : 16] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43, customer_id#55, year_total#56] +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] -(71) TakeOrderedAndProject +(68) CometColumnarToRow [codegen id : 1] Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/extended.txt new file mode 100644 index 0000000000..bcd6c3e73d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 75 out of 75 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/simplified.txt index 7cde4f3773..95b5b340e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/simplified.txt @@ -1,106 +1,70 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] + CometProject [customer_id,customer_first_name,customer_last_name] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/explain.txt index d9f01957c9..f542de162d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/explain.txt @@ -1,754 +1,681 @@ == Physical Plan == -TakeOrderedAndProject (135) -+- * Project (134) - +- * SortMergeJoin Inner (133) - :- * Sort (74) - : +- Exchange (73) - : +- * Filter (72) - : +- * HashAggregate (71) - : +- Exchange (70) - : +- * HashAggregate (69) - : +- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * Project (26) - : : +- * SortMergeJoin LeftOuter (25) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- * Sort (24) - : : +- Exchange (23) - : : +- * Project (22) - : : +- * Filter (21) - : : +- * ColumnarToRow (20) - : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : :- * Project (45) - : : +- * SortMergeJoin LeftOuter (44) - : : :- * Sort (37) - : : : +- Exchange (36) - : : : +- * Project (35) - : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Project (32) - : : : : +- * BroadcastHashJoin Inner BuildRight (31) - : : : : :- * Filter (29) - : : : : : +- * ColumnarToRow (28) - : : : : : +- Scan parquet spark_catalog.default.store_sales (27) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- * Sort (43) - : : +- Exchange (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.store_returns (38) - : +- * Project (64) - : +- * SortMergeJoin LeftOuter (63) - : :- * Sort (56) - : : +- Exchange (55) - : : +- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (51) - : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : :- * Filter (48) - : : : : +- * ColumnarToRow (47) - : : : : +- Scan parquet spark_catalog.default.web_sales (46) - : : : +- ReusedExchange (49) - : : +- ReusedExchange (52) - : +- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * Filter (59) - : +- * ColumnarToRow (58) - : +- Scan parquet spark_catalog.default.web_returns (57) - +- * Sort (132) - +- Exchange (131) - +- * Filter (130) - +- * HashAggregate (129) - +- Exchange (128) - +- * HashAggregate (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- Union (123) - :- * Project (92) - : +- * SortMergeJoin LeftOuter (91) - : :- * Sort (88) - : : +- Exchange (87) - : : +- * Project (86) - : : +- * BroadcastHashJoin Inner BuildRight (85) - : : :- * Project (80) - : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : :- * Filter (77) - : : : : +- * ColumnarToRow (76) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : +- ReusedExchange (78) - : : +- BroadcastExchange (84) - : : +- * Filter (83) - : : +- * ColumnarToRow (82) - : : +- Scan parquet spark_catalog.default.date_dim (81) - : +- * Sort (90) - : +- ReusedExchange (89) - :- * Project (107) - : +- * SortMergeJoin LeftOuter (106) - : :- * Sort (103) - : : +- Exchange (102) - : : +- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Filter (95) - : : : : +- * ColumnarToRow (94) - : : : : +- Scan parquet spark_catalog.default.store_sales (93) - : : : +- ReusedExchange (96) - : : +- ReusedExchange (99) - : +- * Sort (105) - : +- ReusedExchange (104) - +- * Project (122) - +- * SortMergeJoin LeftOuter (121) - :- * Sort (118) - : +- Exchange (117) - : +- * Project (116) - : +- * BroadcastHashJoin Inner BuildRight (115) - : :- * Project (113) - : : +- * BroadcastHashJoin Inner BuildRight (112) - : : :- * Filter (110) - : : : +- * ColumnarToRow (109) - : : : +- Scan parquet spark_catalog.default.web_sales (108) - : : +- ReusedExchange (111) - : +- ReusedExchange (114) - +- * Sort (120) - +- ReusedExchange (119) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometProject (122) + +- CometSortMergeJoin (121) + :- CometSort (66) + : +- CometExchange (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) + +- CometSort (120) + +- CometExchange (119) + +- CometFilter (118) + +- CometHashAggregate (117) + +- CometExchange (116) + +- CometHashAggregate (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometProject (82) + : +- CometSortMergeJoin (81) + : :- CometSort (78) + : : +- CometExchange (77) + : : +- CometProject (76) + : : +- CometBroadcastHashJoin (75) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometFilter (68) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (67) + : : : +- ReusedExchange (69) + : : +- CometBroadcastExchange (74) + : : +- CometFilter (73) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) + : +- CometSort (80) + : +- ReusedExchange (79) + :- CometProject (96) + : +- CometSortMergeJoin (95) + : :- CometSort (92) + : : +- CometExchange (91) + : : +- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometFilter (84) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (83) + : : : +- ReusedExchange (85) + : : +- ReusedExchange (88) + : +- CometSort (94) + : +- ReusedExchange (93) + +- CometProject (110) + +- CometSortMergeJoin (109) + :- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometBroadcastHashJoin (103) + : :- CometProject (101) + : : +- CometBroadcastHashJoin (100) + : : :- CometFilter (98) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (97) + : : +- ReusedExchange (99) + : +- ReusedExchange (102) + +- CometSort (108) + +- ReusedExchange (107) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(8) CometProject Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#12, d_year#13] Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12, d_year#13] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight -(16) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +(13) CometProject Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -(17) Exchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(18) Sort [codegen id : 4] +(15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(21) Filter [codegen id : 5] +(17) CometFilter Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) -(22) Project [codegen id : 5] -Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +(18) CometProject Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -(23) Exchange +(19) CometExchange Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(24) Sort [codegen id : 6] +(20) CometSort Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 7] -Left keys [2]: [cs_order_number#2, cs_item_sk#1] -Right keys [2]: [cr_order_number#15, cr_item_sk#14] -Join type: LeftOuter -Join condition: None +(21) CometSortMergeJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter -(26) Project [codegen id : 7] -Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] +(22) CometProject Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(27) Scan parquet spark_catalog.default.store_sales +(23) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] - -(29) Filter [codegen id : 10] +(24) CometFilter Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_item_sk#21) -(30) ReusedExchange [Reuses operator id: 8] +(25) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(31) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_item_sk#21] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight -(32) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +(27) CometProject Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(33) ReusedExchange [Reuses operator id: 14] +(28) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#31, d_year#32] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#25] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight -(35) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +(30) CometProject Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -(36) Exchange +(31) CometExchange Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) Sort [codegen id : 11] +(32) CometSort Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] -(38) Scan parquet spark_catalog.default.store_returns +(33) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(40) Filter [codegen id : 12] +(34) CometFilter Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) -(41) Project [codegen id : 12] -Output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +(35) CometProject Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -(42) Exchange +(36) CometExchange Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(43) Sort [codegen id : 13] +(37) CometSort Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] -(44) SortMergeJoin [codegen id : 14] -Left keys [2]: [ss_ticket_number#22, ss_item_sk#21] -Right keys [2]: [sr_ticket_number#34, sr_item_sk#33] -Join type: LeftOuter -Join condition: None +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter -(45) Project [codegen id : 14] -Output [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] +(39) CometProject Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -(46) Scan parquet spark_catalog.default.web_sales +(40) CometNativeScan parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] - -(48) Filter [codegen id : 17] +(41) CometFilter Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Condition : isnotnull(ws_item_sk#40) -(49) ReusedExchange [Reuses operator id: 8] +(42) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(50) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#40] -Right keys [1]: [i_item_sk#45] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight -(51) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +(44) CometProject Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(52) ReusedExchange [Reuses operator id: 14] +(45) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#50, d_year#51] -(53) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#44] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight -(54) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +(47) CometProject Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -(55) Exchange +(48) CometExchange Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(56) Sort [codegen id : 18] +(49) CometSort Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST], false, 0 +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] -(57) Scan parquet spark_catalog.default.web_returns +(50) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(59) Filter [codegen id : 19] +(51) CometFilter Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) -(60) Project [codegen id : 19] -Output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +(52) CometProject Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -(61) Exchange +(53) CometExchange Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(62) Sort [codegen id : 20] +(54) CometSort Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST], false, 0 +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] -(63) SortMergeJoin [codegen id : 21] -Left keys [2]: [ws_order_number#41, ws_item_sk#40] -Right keys [2]: [wr_order_number#53, wr_item_sk#52] -Join type: LeftOuter -Join condition: None +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter -(64) Project [codegen id : 21] -Output [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] +(56) CometProject Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] -(65) Union +(57) CometUnion +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] -(66) HashAggregate [codegen id : 22] +(58) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(67) Exchange +(59) CometExchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(68) HashAggregate [codegen id : 23] +(60) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(69) HashAggregate [codegen id : 23] +(61) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#60] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -(70) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(71) HashAggregate [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] +(63) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#63 AS sales_cnt#65, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#66] -(72) Filter [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Condition : isnotnull(sales_cnt#65) +(64) CometFilter +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) -(73) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(65) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(74) Sort [codegen id : 25] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 +(66) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] -(75) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(67) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#71)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(68) CometFilter +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) -(77) Filter [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Condition : isnotnull(cs_item_sk#67) +(69) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(78) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(70) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight -(79) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_item_sk#67] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(71) CometProject +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(80) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Input [10]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(82) ColumnarToRow [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] - -(83) Filter [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +(73) CometFilter +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) -(84) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(74) CometBroadcastExchange +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] -(85) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_sold_date_sk#71] -Right keys [1]: [d_date_sk#77] -Join type: Inner -Join condition: None +(75) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight -(86) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Input [11]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] +(76) CometProject +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] -(87) Exchange -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#68, cs_item_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(77) CometExchange +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(88) Sort [codegen id : 29] -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_order_number#68 ASC NULLS FIRST, cs_item_sk#67 ASC NULLS FIRST], false, 0 +(78) CometSort +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] -(89) ReusedExchange [Reuses operator id: 23] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(79) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] -(90) Sort [codegen id : 31] -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST], false, 0 +(80) CometSort +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] -(91) SortMergeJoin [codegen id : 32] -Left keys [2]: [cs_order_number#68, cs_item_sk#67] -Right keys [2]: [cr_order_number#80, cr_item_sk#79] -Join type: LeftOuter -Join condition: None +(81) CometSortMergeJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter -(92) Project [codegen id : 32] -Output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#69 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#19, (cs_ext_sales_price#70 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(82) CometProject +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] -(93) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(83) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(94) ColumnarToRow [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(84) CometFilter +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) -(95) Filter [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +(85) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(96) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(86) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight -(97) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_item_sk#83] -Right keys [1]: [i_item_sk#88] -Join type: Inner -Join condition: None +(87) CometProject +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(98) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(88) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#89, d_year#90] -(99) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#93, d_year#94] +(89) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_sold_date_sk#87] -Right keys [1]: [d_date_sk#93] -Join type: Inner -Join condition: None +(90) CometProject +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] -(101) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +(91) CometExchange +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(102) Exchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=14] +(92) CometSort +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] -(103) Sort [codegen id : 36] -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST], false, 0 +(93) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] -(104) ReusedExchange [Reuses operator id: 42] -Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +(94) CometSort +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] -(105) Sort [codegen id : 38] -Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] -Arguments: [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST], false, 0 +(95) CometSortMergeJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter -(106) SortMergeJoin [codegen id : 39] -Left keys [2]: [ss_ticket_number#84, ss_item_sk#83] -Right keys [2]: [sr_ticket_number#96, sr_item_sk#95] -Join type: LeftOuter -Join condition: None +(96) CometProject +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] -(107) Project [codegen id : 39] -Output [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#38, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(108) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] +(97) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#103)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(109) ColumnarToRow [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] - -(110) Filter [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Condition : isnotnull(ws_item_sk#99) +(98) CometFilter +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) -(111) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(99) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(112) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_item_sk#99] -Right keys [1]: [i_item_sk#104] -Join type: Inner -Join condition: None +(100) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight -(113) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] -Input [10]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(101) CometProject +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(114) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#109, d_year#110] +(102) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#105, d_year#106] -(115) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_sold_date_sk#103] -Right keys [1]: [d_date_sk#109] -Join type: Inner -Join condition: None +(103) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight -(116) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Input [11]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_date_sk#109, d_year#110] +(104) CometProject +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] -(117) Exchange -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: hashpartitioning(ws_order_number#100, ws_item_sk#99, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(105) CometExchange +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(118) Sort [codegen id : 43] -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: [ws_order_number#100 ASC NULLS FIRST, ws_item_sk#99 ASC NULLS FIRST], false, 0 +(106) CometSort +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] -(119) ReusedExchange [Reuses operator id: 61] -Output [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(107) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] -(120) Sort [codegen id : 45] -Input [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] -Arguments: [wr_order_number#112 ASC NULLS FIRST, wr_item_sk#111 ASC NULLS FIRST], false, 0 +(108) CometSort +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] -(121) SortMergeJoin [codegen id : 46] -Left keys [2]: [ws_order_number#100, ws_item_sk#99] -Right keys [2]: [wr_order_number#112, wr_item_sk#111] -Join type: LeftOuter -Join condition: None +(109) CometSortMergeJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter -(122) Project [codegen id : 46] -Output [7]: [d_year#110, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, (ws_quantity#101 - coalesce(wr_return_quantity#113, 0)) AS sales_cnt#57, (ws_ext_sales_price#102 - coalesce(wr_return_amt#114, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110, wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(110) CometProject +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] -(123) Union +(111) CometUnion +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] -(124) HashAggregate [codegen id : 47] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(112) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(125) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(113) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(126) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(114) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(127) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(115) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#115] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -(128) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=17] +(116) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(129) HashAggregate [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(117) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum(sales_cnt#19)#63 AS sales_cnt#117, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#118] - -(130) Filter [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Condition : isnotnull(sales_cnt#117) - -(131) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(132) Sort [codegen id : 50] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST], false, 0 - -(133) SortMergeJoin [codegen id : 51] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Join type: Inner -Join condition: ((cast(sales_cnt#65 as decimal(17,2)) / cast(sales_cnt#117 as decimal(17,2))) < 0.90000000000000000000) - -(134) Project [codegen id : 51] -Output [10]: [d_year#78 AS prev_year#119, d_year#13 AS year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#117 AS prev_yr_cnt#121, sales_cnt#65 AS curr_yr_cnt#122, (sales_cnt#65 - sales_cnt#117) AS sales_cnt_diff#123, (sales_amt#66 - sales_amt#118) AS sales_amt_diff#124] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] - -(135) TakeOrderedAndProject -Input [10]: [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] -Arguments: 100, [sales_cnt_diff#123 ASC NULLS FIRST], [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] + +(118) CometFilter +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) + +(119) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] + +(120) CometSort +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] + +(121) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) + +(122) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] + +(123) CometTakeOrderedAndProject +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] + +(124) CometColumnarToRow [codegen id : 1] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/extended.txt new file mode 100644 index 0000000000..84620b41ae --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/extended.txt @@ -0,0 +1,158 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_returns + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_returns + +- CometSort + +- CometExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_returns + +Comet accelerated 155 out of 155 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/simplified.txt index f74420a1ad..588e8943c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/simplified.txt @@ -1,232 +1,126 @@ -TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_amt_diff] - WholeStageCodegen (51) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (25) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (24) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (23) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - WholeStageCodegen (22) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (7) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #4 - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (6) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #7 - WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - WholeStageCodegen (14) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (11) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #8 - WholeStageCodegen (10) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (13) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #9 - WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - WholeStageCodegen (21) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #10 - WholeStageCodegen (17) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (20) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #11 - WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (50) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 - WholeStageCodegen (49) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - WholeStageCodegen (48) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 - WholeStageCodegen (47) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (32) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #15 - WholeStageCodegen (28) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (27) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (31) - Sort [cr_order_number,cr_item_sk] - InputAdapter - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - WholeStageCodegen (39) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (36) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 - WholeStageCodegen (35) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (38) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 - WholeStageCodegen (46) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (43) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 - WholeStageCodegen (42) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (45) - Sort [wr_order_number,wr_item_sk] - InputAdapter - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/explain.txt index 68694e2373..f04f5a8a5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/explain.txt @@ -1,218 +1,200 @@ == Physical Plan == -TakeOrderedAndProject (38) -+- * HashAggregate (37) - +- Exchange (36) - +- * HashAggregate (35) - +- Union (34) - :- * Project (15) - : +- * BroadcastHashJoin Inner BuildRight (14) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.date_dim (10) - :- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) - : : : +- Scan parquet spark_catalog.default.web_sales (16) - : : +- ReusedExchange (19) - : +- ReusedExchange (22) - +- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.catalog_sales (25) - : +- ReusedExchange (28) - +- ReusedExchange (31) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (35) ++- CometTakeOrderedAndProject (34) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometUnion (30) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + :- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (14) + : : +- ReusedExchange (16) + : +- ReusedExchange (19) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometFilter (23) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (22) + : +- ReusedExchange (24) + +- ReusedExchange (27) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_category#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#5, i_category#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#5, i_category#6] Condition : isnotnull(i_item_sk#5) -(7) BroadcastExchange +(5) CometProject Input [2]: [i_item_sk#5, i_category#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#5, i_category#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#6, 50, true, false, true) AS i_category#7] + +(6) CometBroadcastExchange +Input [2]: [i_item_sk#5, i_category#7] +Arguments: [i_item_sk#5, i_category#7] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [2]: [i_item_sk#5, i_category#7] +Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight -(9) Project [codegen id : 3] -Output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#7] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] +(10) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : isnotnull(d_date_sk#8) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8, d_year#9, d_qoy#10] -(13) BroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] +Right output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [ss_sold_date_sk#4], [d_date_sk#8], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#7] -Join type: Inner -Join condition: None +(13) CometProject +Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7, d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13], [store AS channel#11, ss_store_sk#2 AS col_name#12, d_year#9, d_qoy#10, i_category#7, ss_ext_sales_price#3 AS ext_sales_price#13] -(15) Project [codegen id : 3] -Output [6]: [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] - -(16) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] +(14) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#16)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#17)] PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] - -(18) Filter [codegen id : 6] -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) +(15) CometFilter +Input [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Condition : (isnull(ws_ship_customer_sk#15) AND isnotnull(ws_item_sk#14)) -(19) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#17, i_category#18] +(16) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#18, i_category#19] -(20) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#13] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Right output [2]: [i_item_sk#18, i_category#19] +Arguments: [ws_item_sk#14], [i_item_sk#18], Inner, BuildRight -(21) Project [codegen id : 6] -Output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] +(18) CometProject +Input [6]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_item_sk#18, i_category#19] +Arguments: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19], [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] -(22) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] +(19) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#20, d_year#21, d_qoy#22] -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#16] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] +Right output [3]: [d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight -(24) Project [codegen id : 6] -Output [6]: [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] +(21) CometProject +Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] -(25) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] +(22) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#28)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 9] -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] - -(27) Filter [codegen id : 9] -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) - -(28) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#29, i_category#30] - -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_item_sk#26] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 9] -Output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] - -(31) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_sold_date_sk#28] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [6]: [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] - -(34) Union - -(35) HashAggregate [codegen id : 10] -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count#37, sum#38] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] - -(36) Exchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(37) HashAggregate [codegen id : 11] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count(1)#41, sum(UnscaledValue(ext_sales_price#12))#42] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count(1)#41 AS sales_cnt#43, MakeDecimal(sum(UnscaledValue(ext_sales_price#12))#42,17,2) AS sales_amt#44] - -(38) TakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] -Arguments: 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] +(23) CometFilter +Input [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Condition : (isnull(cs_ship_addr_sk#26) AND isnotnull(cs_item_sk#27)) + +(24) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#30, i_category#31] + +(25) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Right output [2]: [i_item_sk#30, i_category#31] +Arguments: [cs_item_sk#27], [i_item_sk#30], Inner, BuildRight + +(26) CometProject +Input [6]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29, i_item_sk#30, i_category#31] +Arguments: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31], [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] + +(27) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#32, d_year#33, d_qoy#34] + +(28) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] +Right output [3]: [d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [cs_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight + +(29) CometProject +Input [7]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31, d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37], [catalog AS channel#35, cs_ship_addr_sk#26 AS col_name#36, d_year#33, d_qoy#34, i_category#31, cs_ext_sales_price#28 AS ext_sales_price#37] + +(30) CometUnion +Child 0 Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Child 1 Input [6]: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25] +Child 2 Input [6]: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37] + +(31) CometHashAggregate +Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#13))] + +(32) CometExchange +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Arguments: hashpartitioning(channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(33) CometHashAggregate +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#13))] + +(34) CometTakeOrderedAndProject +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#11 ASC NULLS FIRST,col_name#12 ASC NULLS FIRST,d_year#9 ASC NULLS FIRST,d_qoy#10 ASC NULLS FIRST,i_category#7 ASC NULLS FIRST], output=[channel#11,col_name#12,d_year#9,d_qoy#10,i_category#7,sales_cnt#40,sales_amt#41]), [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41], 100, 0, [channel#11 ASC NULLS FIRST, col_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, i_category#7 ASC NULLS FIRST], [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] + +(35) CometColumnarToRow [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/extended.txt new file mode 100644 index 0000000000..cbb627c528 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/extended.txt @@ -0,0 +1,47 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 44 out of 44 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/simplified.txt index 2d208fdbb2..abb482c3d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/simplified.txt @@ -1,58 +1,37 @@ -TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - WholeStageCodegen (11) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum] - InputAdapter - Exchange [channel,col_name,d_year,d_qoy,i_category] #1 - WholeStageCodegen (10) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum] - InputAdapter - Union - WholeStageCodegen (3) - Project [ss_store_sk,d_year,d_qoy,i_category,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - WholeStageCodegen (6) - Project [ws_ship_customer_sk,d_year,d_qoy,i_category,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_ship_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_category] #2 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - WholeStageCodegen (9) - Project [cs_ship_addr_sk,d_year,d_qoy,i_category,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_ship_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_category] #2 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] + CometHashAggregate [count,sum] [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count(1),sum(UnscaledValue(ext_sales_price))] + CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 + CometHashAggregate [ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,count,sum] + CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category] #2 + CometProject [i_category] [i_item_sk,i_category] + CometFilter [i_item_sk,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_category] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] + CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] + CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/explain.txt index 6d88eef211..476f7e4df0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/explain.txt @@ -1,527 +1,475 @@ == Physical Plan == -TakeOrderedAndProject (89) -+- * HashAggregate (88) - +- Exchange (87) - +- * HashAggregate (86) - +- * Expand (85) - +- Union (84) - :- * Project (34) - : +- * BroadcastHashJoin LeftOuter BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.store_returns (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - :- * Project (53) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (52) - : :- BroadcastExchange (43) - : : +- * HashAggregate (42) - : : +- Exchange (41) - : : +- * HashAggregate (40) - : : +- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * ColumnarToRow (36) - : : : +- Scan parquet spark_catalog.default.catalog_sales (35) - : : +- ReusedExchange (37) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * ColumnarToRow (45) - : : +- Scan parquet spark_catalog.default.catalog_returns (44) - : +- ReusedExchange (46) - +- * Project (83) - +- * BroadcastHashJoin LeftOuter BuildRight (82) - :- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- * Project (65) - : +- * BroadcastHashJoin Inner BuildRight (64) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- * Filter (56) - : : : +- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (57) - : +- BroadcastExchange (63) - : +- * Filter (62) - : +- * ColumnarToRow (61) - : +- Scan parquet spark_catalog.default.web_page (60) - +- BroadcastExchange (81) - +- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- * Project (77) - +- * BroadcastHashJoin Inner BuildRight (76) - :- * Project (74) - : +- * BroadcastHashJoin Inner BuildRight (73) - : :- * Filter (71) - : : +- * ColumnarToRow (70) - : : +- Scan parquet spark_catalog.default.web_returns (69) - : +- ReusedExchange (72) - +- ReusedExchange (75) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (85) ++- * HashAggregate (84) + +- * CometColumnarToRow (83) + +- CometColumnarExchange (82) + +- * HashAggregate (81) + +- * Expand (80) + +- Union (79) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (28) + : +- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (21) + : : +- CometBroadcastHashJoin (20) + : : :- CometFilter (18) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (17) + : : +- ReusedExchange (19) + : +- ReusedExchange (22) + :- * Project (50) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : :- BroadcastExchange (40) + : : +- * CometColumnarToRow (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (32) + : : +- ReusedExchange (33) + : +- * CometColumnarToRow (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometNativeScan parquet spark_catalog.default.catalog_returns (41) + : +- ReusedExchange (42) + +- * CometColumnarToRow (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometFilter (52) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (51) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometNativeScan parquet spark_catalog.default.web_page (56) + +- CometBroadcastExchange (75) + +- CometHashAggregate (74) + +- CometExchange (73) + +- CometHashAggregate (72) + +- CometProject (71) + +- CometBroadcastHashJoin (70) + :- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometFilter (65) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (64) + : +- ReusedExchange (66) + +- ReusedExchange (69) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_date#6] Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +(8) CometProject Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [s_store_sk#7] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [1]: [s_store_sk#7] Condition : isnotnull(s_store_sk#7) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#7] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +(13) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -(17) HashAggregate [codegen id : 3] +(14) CometHashAggregate Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] -(18) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 8] -Input [3]: [s_store_sk#7, sum#10, sum#11] +(16) CometHashAggregate +Input [3]: [s_store_sk#7, sum#8, sum#9] Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] -(20) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(17) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#19)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(18) CometFilter +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) -(22) Filter [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) +(19) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#14] -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] +(20) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_returned_date_sk#19] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None +(21) CometProject +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] -(25) Project [codegen id : 6] -Output [3]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18] -Input [5]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19, d_date_sk#20] +(22) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#15] -(26) ReusedExchange [Reuses operator id: 14] -Output [1]: [s_store_sk#21] +(23) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_store_sk#16] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None +(24) CometProject +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] -(28) Project [codegen id : 6] -Output [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, s_store_sk#21] - -(29) HashAggregate [codegen id : 6] -Input [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#17)), partial_sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum#22, sum#23] -Results [3]: [s_store_sk#21, sum#24, sum#25] - -(30) Exchange -Input [3]: [s_store_sk#21, sum#24, sum#25] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [s_store_sk#21, sum#24, sum#25] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#17)), sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#17))#26, sum(UnscaledValue(sr_net_loss#18))#27] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#17))#26,17,2) AS returns#28, MakeDecimal(sum(UnscaledValue(sr_net_loss#18))#27,17,2) AS profit_loss#29] - -(32) BroadcastExchange -Input [3]: [s_store_sk#21, returns#28, profit_loss#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#21] -Join type: LeftOuter -Join condition: None +(25) CometHashAggregate +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] + +(26) CometExchange +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(27) CometHashAggregate +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] + +(28) CometBroadcastExchange +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] -(34) Project [codegen id : 8] -Output [5]: [sales#14, coalesce(returns#28, 0.00) AS returns#30, (profit#15 - coalesce(profit_loss#29, 0.00)) AS profit#31, store channel AS channel#32, s_store_sk#7 AS id#33] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#21, returns#28, profit_loss#29] +(29) CometBroadcastHashJoin +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight -(35) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(30) CometProject +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [sales#20, returns#22, profit#23, channel#24, id#25], [sales#20, coalesce(returns#18, 0.00) AS returns#22, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#23, store channel AS channel#24, s_store_sk#7 AS id#25] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] + +(32) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#37)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(33) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] -(37) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#38] +(34) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None +(35) CometProject +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] + +(36) CometHashAggregate +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] + +(37) CometExchange +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometHashAggregate +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] + +(39) CometColumnarToRow [codegen id : 2] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] + +(40) BroadcastExchange +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] +Arguments: IdentityBroadcastMode, [plan_id=4] -(39) Project [codegen id : 10] -Output [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Input [5]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37, d_date_sk#38] - -(40) HashAggregate [codegen id : 10] -Input [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#35)), partial_sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum#39, sum#40] -Results [3]: [cs_call_center_sk#34, sum#41, sum#42] - -(41) Exchange -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Arguments: hashpartitioning(cs_call_center_sk#34, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#35)), sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#35))#43, sum(UnscaledValue(cs_net_profit#36))#44] -Results [3]: [cs_call_center_sk#34, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#35))#43,17,2) AS sales#45, MakeDecimal(sum(UnscaledValue(cs_net_profit#36))#44,17,2) AS profit#46] - -(43) BroadcastExchange -Input [3]: [cs_call_center_sk#34, sales#45, profit#46] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(44) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] +(41) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#49)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 13] -Input [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] +(42) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#38] -(46) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#50] +(43) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [cr_returned_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(44) CometProject +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] -(48) Project [codegen id : 13] -Output [2]: [cr_return_amount#47, cr_net_loss#48] -Input [4]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49, d_date_sk#50] - -(49) HashAggregate [codegen id : 13] -Input [2]: [cr_return_amount#47, cr_net_loss#48] +(45) CometHashAggregate +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#47)), partial_sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [2]: [sum#53, sum#54] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] -(50) Exchange -Input [2]: [sum#53, sum#54] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [2]: [sum#39, sum#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(51) HashAggregate -Input [2]: [sum#53, sum#54] +(47) CometHashAggregate +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#47)), sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#47))#55, sum(UnscaledValue(cr_net_loss#48))#56] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#47))#55,17,2) AS returns#57, MakeDecimal(sum(UnscaledValue(cr_net_loss#48))#56,17,2) AS profit_loss#58] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] + +(48) CometColumnarToRow +Input [2]: [returns#41, profit_loss#42] -(52) BroadcastNestedLoopJoin [codegen id : 14] +(49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None -(53) Project [codegen id : 14] -Output [5]: [sales#45, returns#57, (profit#46 - profit_loss#58) AS profit#59, catalog channel AS channel#60, cs_call_center_sk#34 AS id#61] -Input [5]: [cs_call_center_sk#34, sales#45, profit#46, returns#57, profit_loss#58] +(50) Project [codegen id : 3] +Output [5]: [sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#43, catalog channel AS channel#44, cs_call_center_sk#26 AS id#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] +(51) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#65)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] - -(56) Filter [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Condition : isnotnull(ws_web_page_sk#62) +(52) CometFilter +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) -(57) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#66] +(53) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#50] -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None +(54) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight -(59) Project [codegen id : 17] -Output [3]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64] -Input [5]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65, d_date_sk#66] +(55) CometProject +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] -(60) Scan parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#67] +(56) CometNativeScan parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [1]: [wp_web_page_sk#67] - -(62) Filter [codegen id : 16] -Input [1]: [wp_web_page_sk#67] -Condition : isnotnull(wp_web_page_sk#67) - -(63) BroadcastExchange -Input [1]: [wp_web_page_sk#67] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(64) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_web_page_sk#62] -Right keys [1]: [wp_web_page_sk#67] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 17] -Output [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] - -(66) HashAggregate [codegen id : 17] -Input [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#63)), partial_sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum#68, sum#69] -Results [3]: [wp_web_page_sk#67, sum#70, sum#71] - -(67) Exchange -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Arguments: hashpartitioning(wp_web_page_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#63)), sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#63))#72, sum(UnscaledValue(ws_net_profit#64))#73] -Results [3]: [wp_web_page_sk#67, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#63))#72,17,2) AS sales#74, MakeDecimal(sum(UnscaledValue(ws_net_profit#64))#73,17,2) AS profit#75] - -(69) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#79)] -PushedFilters: [IsNotNull(wr_web_page_sk)] -ReadSchema: struct - -(70) ColumnarToRow [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] +(57) CometFilter +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) -(71) Filter [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Condition : isnotnull(wr_web_page_sk#76) +(58) CometBroadcastExchange +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] -(72) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#80] +(59) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight -(73) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_returned_date_sk#79] -Right keys [1]: [d_date_sk#80] -Join type: Inner -Join condition: None +(60) CometProject +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] -(74) Project [codegen id : 20] -Output [3]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78] -Input [5]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79, d_date_sk#80] +(61) CometHashAggregate +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] -(75) ReusedExchange [Reuses operator id: 63] -Output [1]: [wp_web_page_sk#81] +(62) CometExchange +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(76) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_web_page_sk#76] -Right keys [1]: [wp_web_page_sk#81] -Join type: Inner -Join condition: None +(63) CometHashAggregate +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] -(77) Project [codegen id : 20] -Output [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] - -(78) HashAggregate [codegen id : 20] -Input [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#77)), partial_sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum#82, sum#83] -Results [3]: [wp_web_page_sk#81, sum#84, sum#85] - -(79) Exchange -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Arguments: hashpartitioning(wp_web_page_sk#81, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 21] -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [sum(UnscaledValue(wr_return_amt#77)), sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#77))#86, sum(UnscaledValue(wr_net_loss#78))#87] -Results [3]: [wp_web_page_sk#81, MakeDecimal(sum(UnscaledValue(wr_return_amt#77))#86,17,2) AS returns#88, MakeDecimal(sum(UnscaledValue(wr_net_loss#78))#87,17,2) AS profit_loss#89] - -(81) BroadcastExchange -Input [3]: [wp_web_page_sk#81, returns#88, profit_loss#89] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(82) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [wp_web_page_sk#67] -Right keys [1]: [wp_web_page_sk#81] -Join type: LeftOuter -Join condition: None +(64) CometNativeScan parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct -(83) Project [codegen id : 22] -Output [5]: [sales#74, coalesce(returns#88, 0.00) AS returns#90, (profit#75 - coalesce(profit_loss#89, 0.00)) AS profit#91, web channel AS channel#92, wp_web_page_sk#67 AS id#93] -Input [6]: [wp_web_page_sk#67, sales#74, profit#75, wp_web_page_sk#81, returns#88, profit_loss#89] - -(84) Union - -(85) Expand [codegen id : 23] -Input [5]: [sales#14, returns#30, profit#31, channel#32, id#33] -Arguments: [[sales#14, returns#30, profit#31, channel#32, id#33, 0], [sales#14, returns#30, profit#31, channel#32, null, 1], [sales#14, returns#30, profit#31, null, null, 3]], [sales#14, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] - -(86) HashAggregate [codegen id : 23] -Input [6]: [sales#14, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#30), partial_sum(profit#31)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(87) Exchange -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#94, id#95, spark_grouping_id#96, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(88) HashAggregate [codegen id : 24] -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [sum(sales#14), sum(returns#30), sum(profit#31)] -Aggregate Attributes [3]: [sum(sales#14)#109, sum(returns#30)#110, sum(profit#31)#111] -Results [5]: [channel#94, id#95, sum(sales#14)#109 AS sales#112, sum(returns#30)#110 AS returns#113, sum(profit#31)#111 AS profit#114] - -(89) TakeOrderedAndProject -Input [5]: [channel#94, id#95, sales#112, returns#113, profit#114] -Arguments: 100, [channel#94 ASC NULLS FIRST, id#95 ASC NULLS FIRST], [channel#94, id#95, sales#112, returns#113, profit#114] +(65) CometFilter +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) + +(66) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(67) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(68) CometProject +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] + +(69) ReusedExchange [Reuses operator id: 58] +Output [1]: [wp_web_page_sk#59] + +(70) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(71) CometProject +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] + +(72) CometHashAggregate +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] + +(73) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(74) CometHashAggregate +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] + +(75) CometBroadcastExchange +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] + +(76) CometBroadcastHashJoin +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight + +(77) CometProject +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [sales#64, returns#66, profit#67, channel#68, id#69], [sales#64, coalesce(returns#62, 0.00) AS returns#66, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#67, web channel AS channel#68, wp_web_page_sk#51 AS id#69] + +(78) CometColumnarToRow [codegen id : 4] +Input [5]: [sales#64, returns#66, profit#67, channel#68, id#69] + +(79) Union + +(80) Expand [codegen id : 5] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] +Arguments: [[sales#20, returns#22, profit#23, channel#24, id#25, 0], [sales#20, returns#22, profit#23, channel#24, null, 1], [sales#20, returns#22, profit#23, null, null, 3]], [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] + +(81) HashAggregate [codegen id : 5] +Input [6]: [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#22), partial_sum(profit#23)] +Aggregate Attributes [6]: [sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78] +Results [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] + +(82) CometColumnarExchange +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Arguments: hashpartitioning(channel#70, id#71, spark_grouping_id#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(83) CometColumnarToRow [codegen id : 6] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] + +(84) HashAggregate [codegen id : 6] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [sum(sales#20), sum(returns#22), sum(profit#23)] +Aggregate Attributes [3]: [sum(sales#20)#85, sum(returns#22)#86, sum(profit#23)#87] +Results [5]: [channel#70, id#71, sum(sales#20)#85 AS sales#88, sum(returns#22)#86 AS returns#89, sum(profit#23)#87 AS profit#90] + +(85) TakeOrderedAndProject +Input [5]: [channel#70, id#71, sales#88, returns#89, profit#90] +Arguments: 100, [channel#70 ASC NULLS FIRST, id#71 ASC NULLS FIRST], [channel#70, id#71, sales#88, returns#89, profit#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/extended.txt new file mode 100644 index 0000000000..f857003b9a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/extended.txt @@ -0,0 +1,106 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Union + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- Project + : +- BroadcastNestedLoopJoin + : :- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometColumnarToRow + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 91 out of 99 eligible operators (91%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/simplified.txt index b813daff14..165dc44350 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/simplified.txt @@ -1,136 +1,98 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (24) + WholeStageCodegen (6) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (23) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (8) - Project [sales,returns,profit,profit_loss,s_store_sk] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [s_store_sk] #2 - WholeStageCodegen (3) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,s_store_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [s_store_sk] #6 - WholeStageCodegen (6) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - Project [sr_return_amt,sr_net_loss,s_store_sk] - BroadcastHashJoin [sr_store_sk,s_store_sk] - Project [sr_store_sk,sr_return_amt,sr_net_loss] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [s_store_sk] #4 - WholeStageCodegen (14) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (5) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #8 - WholeStageCodegen (10) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter + CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [s_store_sk,sales,profit,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #2 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #4 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #5 + CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #6 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] ReusedExchange [d_date_sk] #3 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + ReusedExchange [s_store_sk] #4 + WholeStageCodegen (3) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - Project [cr_return_amount,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #8 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #9 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #3 - WholeStageCodegen (22) - Project [sales,returns,profit,profit_loss,wp_web_page_sk] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #10 - WholeStageCodegen (17) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - BroadcastExchange #12 - WholeStageCodegen (21) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #13 - WholeStageCodegen (20) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - Project [wr_return_amt,wr_net_loss,wp_web_page_sk] - BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] - Project [wr_web_page_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [wp_web_page_sk] #11 + CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #10 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #12 + CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #13 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/explain.txt index 430c1c511e..fdcf5824de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/explain.txt @@ -1,417 +1,373 @@ == Physical Plan == -TakeOrderedAndProject (73) -+- * Project (72) - +- * SortMergeJoin Inner (71) - :- * Project (48) - : +- * SortMergeJoin Inner (47) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Project (14) - : : : +- * Filter (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- * Sort (46) - : +- * Filter (45) - : +- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * Project (41) - : +- * BroadcastHashJoin Inner BuildRight (40) - : :- * Project (38) - : : +- * Filter (37) - : : +- * SortMergeJoin LeftOuter (36) - : : :- * Sort (29) - : : : +- Exchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.web_sales (25) - : : +- * Sort (35) - : : +- Exchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.web_returns (30) - : +- ReusedExchange (39) - +- * Sort (70) - +- * Filter (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * Filter (61) - : +- * SortMergeJoin LeftOuter (60) - : :- * Sort (53) - : : +- Exchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.catalog_sales (49) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- Scan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (67) ++- * Project (66) + +- * CometColumnarToRow (65) + +- CometSortMergeJoin (64) + :- CometProject (43) + : +- CometSortMergeJoin (42) + : :- CometSort (21) + : : +- CometHashAggregate (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : +- CometSort (41) + : +- CometFilter (40) + : +- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (33) + : : +- CometFilter (32) + : : +- CometSortMergeJoin (31) + : : :- CometSort (25) + : : : +- CometExchange (24) + : : : +- CometFilter (23) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (22) + : : +- CometSort (30) + : : +- CometExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (26) + : +- ReusedExchange (34) + +- CometSort (63) + +- CometFilter (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometFilter (54) + : +- CometSortMergeJoin (53) + : :- CometSort (47) + : : +- CometExchange (46) + : : +- CometFilter (45) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (44) + : +- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometFilter (49) + : +- CometNativeScan parquet spark_catalog.default.catalog_returns (48) + +- ReusedExchange (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#8, sr_ticket_number#9] +(7) CometProject Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] -(10) Exchange +(8) CometExchange Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#9, sr_item_sk#8] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter -(13) Filter [codegen id : 6] +(11) CometFilter Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Condition : isnull(sr_ticket_number#9) -(14) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +(12) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(15) Scan parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] - -(17) Filter [codegen id : 5] +(14) CometFilter Input [2]: [d_date_sk#11, d_year#12] Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) -(18) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [d_date_sk#11, d_year#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#11, d_year#12] -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight -(20) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +(17) CometProject Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -(22) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) HashAggregate [codegen id : 7] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] +(20) CometHashAggregate +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] -(24) Sort [codegen id : 7] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 +(21) CometSort +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(25) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(22) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#32)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(23) CometFilter +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) -(27) Filter [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Condition : (isnotnull(ws_item_sk#26) AND isnotnull(ws_bill_customer_sk#27)) +(24) CometExchange +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(28) Exchange -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: hashpartitioning(ws_order_number#28, ws_item_sk#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometSort +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(29) Sort [codegen id : 9] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: [ws_order_number#28 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] +(26) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(32) Filter [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Condition : (isnotnull(wr_order_number#34) AND isnotnull(wr_item_sk#33)) - -(33) Project [codegen id : 10] -Output [2]: [wr_item_sk#33, wr_order_number#34] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(34) Exchange -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: hashpartitioning(wr_order_number#34, wr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 11] -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: [wr_order_number#34 ASC NULLS FIRST, wr_item_sk#33 ASC NULLS FIRST], false, 0 - -(36) SortMergeJoin [codegen id : 13] -Left keys [2]: [ws_order_number#28, ws_item_sk#26] -Right keys [2]: [wr_order_number#34, wr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(37) Filter [codegen id : 13] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] -Condition : isnull(wr_order_number#34) - -(38) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] - -(39) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#36, d_year#37] - -(40) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#32] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Input [8]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, d_date_sk#36, d_year#37] - -(42) HashAggregate [codegen id : 13] -Input [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [partial_sum(ws_quantity#29), partial_sum(UnscaledValue(ws_wholesale_cost#30)), partial_sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum#38, sum#39, sum#40] -Results [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] - -(43) Exchange -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Arguments: hashpartitioning(d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(44) HashAggregate [codegen id : 14] -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [sum(ws_quantity#29), sum(UnscaledValue(ws_wholesale_cost#30)), sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum(ws_quantity#29)#44, sum(UnscaledValue(ws_wholesale_cost#30))#45, sum(UnscaledValue(ws_sales_price#31))#46] -Results [6]: [d_year#37 AS ws_sold_year#47, ws_item_sk#26, ws_bill_customer_sk#27 AS ws_customer_sk#48, sum(ws_quantity#29)#44 AS ws_qty#49, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#30))#45,17,2) AS ws_wc#50, MakeDecimal(sum(UnscaledValue(ws_sales_price#31))#46,17,2) AS ws_sp#51] - -(45) Filter [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Condition : (coalesce(ws_qty#49, 0) > 0) - -(46) Sort [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Arguments: [ws_sold_year#47 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST, ws_customer_sk#48 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 15] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 15] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] - -(49) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] +(27) CometFilter +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) + +(28) CometProject +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] + +(29) CometExchange +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(30) CometSort +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] + +(31) CometSortMergeJoin +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter + +(32) CometFilter +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) + +(33) CometProject +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] + +(34) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#30, d_year#31] + +(35) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight + +(36) CometProject +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] + +(37) CometHashAggregate +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] + +(38) CometExchange +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometHashAggregate +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] + +(40) CometFilter +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) + +(41) CometSort +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] + +(42) CometSortMergeJoin +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner + +(43) CometProject +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] + +(44) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] - -(51) Filter [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Condition : (isnotnull(cs_item_sk#53) AND isnotnull(cs_bill_customer_sk#52)) +(45) CometFilter +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) -(52) Exchange -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: hashpartitioning(cs_order_number#54, cs_item_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(53) Sort [codegen id : 17] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: [cs_order_number#54 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST], false, 0 +(47) CometSort +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] -(54) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] +(48) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(56) Filter [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Condition : (isnotnull(cr_order_number#60) AND isnotnull(cr_item_sk#59)) - -(57) Project [codegen id : 18] -Output [2]: [cr_item_sk#59, cr_order_number#60] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(58) Exchange -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: hashpartitioning(cr_order_number#60, cr_item_sk#59, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(59) Sort [codegen id : 19] -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: [cr_order_number#60 ASC NULLS FIRST, cr_item_sk#59 ASC NULLS FIRST], false, 0 - -(60) SortMergeJoin [codegen id : 21] -Left keys [2]: [cs_order_number#54, cs_item_sk#53] -Right keys [2]: [cr_order_number#60, cr_item_sk#59] -Join type: LeftOuter -Join condition: None - -(61) Filter [codegen id : 21] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] -Condition : isnull(cr_order_number#60) - -(62) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] - -(63) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#62, d_year#63] - -(64) BroadcastHashJoin [codegen id : 21] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Input [8]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, d_date_sk#62, d_year#63] - -(66) HashAggregate [codegen id : 21] -Input [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [partial_sum(cs_quantity#55), partial_sum(UnscaledValue(cs_wholesale_cost#56)), partial_sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum#64, sum#65, sum#66] -Results [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] - -(67) Exchange -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Arguments: hashpartitioning(d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [sum(cs_quantity#55), sum(UnscaledValue(cs_wholesale_cost#56)), sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum(cs_quantity#55)#70, sum(UnscaledValue(cs_wholesale_cost#56))#71, sum(UnscaledValue(cs_sales_price#57))#72] -Results [6]: [d_year#63 AS cs_sold_year#73, cs_item_sk#53, cs_bill_customer_sk#52 AS cs_customer_sk#74, sum(cs_quantity#55)#70 AS cs_qty#75, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#56))#71,17,2) AS cs_wc#76, MakeDecimal(sum(UnscaledValue(cs_sales_price#57))#72,17,2) AS cs_sp#77] - -(69) Filter [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Condition : (coalesce(cs_qty#75, 0) > 0) - -(70) Sort [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Arguments: [cs_sold_year#73 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST, cs_customer_sk#74 ASC NULLS FIRST], false, 0 - -(71) SortMergeJoin [codegen id : 23] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74] -Join type: Inner -Join condition: None - -(72) Project [codegen id : 23] -Output [12]: [round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) AS ratio#78, ss_qty#23 AS store_qty#79, ss_wc#24 AS store_wholesale_cost#80, ss_sp#25 AS store_sales_price#81, (coalesce(ws_qty#49, 0) + coalesce(cs_qty#75, 0)) AS other_chan_qty#82, (coalesce(ws_wc#50, 0.00) + coalesce(cs_wc#76, 0.00)) AS other_chan_wholesale_cost#83, (coalesce(ws_sp#51, 0.00) + coalesce(cs_sp#77, 0.00)) AS other_chan_sales_price#84, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, cs_qty#75] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51, cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] - -(73) TakeOrderedAndProject -Input [12]: [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, cs_qty#75] -Arguments: 100, [ratio#78 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#82 ASC NULLS FIRST, other_chan_wholesale_cost#83 ASC NULLS FIRST, other_chan_sales_price#84 ASC NULLS FIRST, round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) ASC NULLS FIRST], [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84] +(49) CometFilter +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) + +(50) CometProject +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] + +(51) CometExchange +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(52) CometSort +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] + +(53) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter + +(54) CometFilter +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) + +(55) CometProject +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] + +(56) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#50, d_year#51] + +(57) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight + +(58) CometProject +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] + +(59) CometHashAggregate +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] + +(60) CometExchange +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(61) CometHashAggregate +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] + +(62) CometFilter +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) + +(63) CometSort +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] + +(64) CometSortMergeJoin +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner + +(65) CometColumnarToRow [codegen id : 1] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(66) Project [codegen id : 1] +Output [12]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(67) TakeOrderedAndProject +Input [12]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Arguments: 100, [ratio#60 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, round((cast(ss_qty#17 as double) / cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)), 2) ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/extended.txt new file mode 100644 index 0000000000..f180fb1878 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/extended.txt @@ -0,0 +1,73 @@ + TakeOrderedAndProject [COMET: ] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding] + +- CometColumnarToRow + +- CometSortMergeJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 68 out of 70 eligible operators (97%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/simplified.txt index 839bcca08a..4793204ab0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/simplified.txt @@ -1,123 +1,69 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (23) + WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] + CometColumnarToRow InputAdapter - WholeStageCodegen (15) - Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] - InputAdapter - WholeStageCodegen (7) - Sort [ss_sold_year,ss_item_sk,ss_customer_sk] - HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - Filter [sr_ticket_number] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (14) - Sort [ws_sold_year,ws_item_sk,ws_customer_sk] - Filter [ws_qty] - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - WholeStageCodegen (13) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - Filter [wr_order_number] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (9) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #6 - WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (11) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #7 - WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 - InputAdapter - WholeStageCodegen (22) - Sort [cs_sold_year,cs_item_sk,cs_customer_sk] - Filter [cs_qty] - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - WholeStageCodegen (21) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - Filter [cr_order_number] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (17) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #9 - WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (19) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #10 - WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 + CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/explain.txt index dd7510c121..eacdb29f86 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/explain.txt @@ -1,198 +1,179 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (31) - +- * Filter (30) - +- * ColumnarToRow (29) - +- Scan parquet spark_catalog.default.customer (28) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (27) + +- CometProject (26) + +- CometFilter (25) + +- CometNativeScan parquet spark_catalog.default.customer (24) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(5) CometProject Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#9] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(10) Project [codegen id : 4] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(8) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) -(14) Project [codegen id : 2] -Output [2]: [s_store_sk#12, s_city#14] +(11) CometProject Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [s_store_sk#12, s_city#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#12, s_city#14] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [2]: [s_store_sk#12, s_city#14] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight -(17) Project [codegen id : 4] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +(14) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#15] +(17) CometProject Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#15] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight -(24) Project [codegen id : 4] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +(20) CometProject Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#18, sum#19] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -(26) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] +(23) CometHashAggregate +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#22, sum(UnscaledValue(ss_net_profit#7))#23] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#22,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#23,17,2) AS profit#25] -(28) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] +(24) CometNativeScan parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] +(25) CometFilter +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Condition : isnotnull(c_customer_sk#20) + +(26) CometProject +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24], [c_customer_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#21, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#22, 30, true, false, true) AS c_last_name#24] -(30) Filter [codegen id : 5] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Condition : isnotnull(c_customer_sk#26) +(27) CometBroadcastExchange +Input [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24] -(31) BroadcastExchange -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastHashJoin +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26] +Right output [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [ss_customer_sk#1], [c_customer_sk#20], Inner, BuildRight -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#26] -Join type: Inner -Join condition: None +(29) CometProject +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26, c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14], [c_last_name#24, c_first_name#23, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] -(33) Project [codegen id : 6] -Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#24, profit#25, c_customer_sk#26, c_first_name#27, c_last_name#28] +(30) CometTakeOrderedAndProject +Input [7]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#26 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,substr(s_city, 1, 30)#27,ss_ticket_number#5,amt#25,profit#26]), [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26], 100, 0, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#26 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] -(34) TakeOrderedAndProject -Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#25 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25] +(31) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/extended.txt new file mode 100644 index 0000000000..178dbb8e72 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/extended.txt @@ -0,0 +1,33 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 30 out of 30 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/simplified.txt index 138ec9eb1e..7612d7f5ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/simplified.txt @@ -1,50 +1,33 @@ -TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt] - WholeStageCodegen (6) - Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk,s_city] - Filter [s_number_employees,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] + CometProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,s_city,amt,profit,c_customer_sk,c_first_name,c_last_name] + CometHashAggregate [ss_addr_sk,sum,sum] [ss_ticket_number,ss_customer_sk,s_city,amt,profit,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk,s_city] #3 + CometProject [s_store_sk,s_city] + CometFilter [s_store_sk,s_number_employees,s_city] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/explain.txt index 1abf0a4f8e..619df5d251 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/explain.txt @@ -1,278 +1,251 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * BroadcastHashJoin LeftSemi BuildRight (37) - :- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.customer_address (17) - +- BroadcastExchange (36) - +- * Project (35) - +- * Filter (34) - +- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.customer_address (21) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.customer (24) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (37) + +- CometHashAggregate (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometBroadcastHashJoin (33) + :- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.customer_address (15) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometFilter (19) + : +- CometNativeScan parquet spark_catalog.default.customer_address (18) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.customer (21) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] +(8) CometProject Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(10) CometFilter Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Condition : (isnotnull(s_store_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true))) -(13) Filter [codegen id : 2] +(11) CometProject Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true) AS s_zip#10] -(14) BroadcastExchange -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] -(15) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_store_sk#1, ss_net_profit#2] +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(16) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] +(14) CometProject +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] -(17) Scan parquet spark_catalog.default.customer_address -Output [1]: [ca_zip#10] +(15) CometNativeScan parquet spark_catalog.default.customer_address +Output [1]: [ca_zip#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] -Input [1]: [ca_zip#10] - -(19) Filter [codegen id : 6] -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) +(16) CometFilter +Input [1]: [ca_zip#11] +Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5))) -(20) Project [codegen id : 6] -Output [1]: [substr(ca_zip#10, 1, 5) AS ca_zip#11] -Input [1]: [ca_zip#10] +(17) CometProject +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) AS ca_zip#12] -(21) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#12, ca_zip#13] +(18) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#12, ca_zip#13] +(19) CometFilter +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) -(23) Filter [codegen id : 4] -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) +(20) CometProject +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#14, 10, true, false, true) AS ca_zip#15] -(24) Scan parquet spark_catalog.default.customer -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] +(21) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)] +PushedFilters: [IsNotNull(c_current_addr_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 3] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] - -(26) Filter [codegen id : 3] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) +(22) CometFilter +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#17, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#16)) -(27) Project [codegen id : 3] -Output [1]: [c_current_addr_sk#14] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] +(23) CometProject +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] -(28) BroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(24) CometBroadcastExchange +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] -(29) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ca_address_sk#12] -Right keys [1]: [c_current_addr_sk#14] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight -(30) Project [codegen id : 4] -Output [1]: [ca_zip#13] -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] +(26) CometProject +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] -(31) HashAggregate [codegen id : 4] -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +(27) CometHashAggregate +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#16] -Results [2]: [ca_zip#13, count#17] -(32) Exchange -Input [2]: [ca_zip#13, count#17] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 5] -Input [2]: [ca_zip#13, count#17] -Keys [1]: [ca_zip#13] +(29) CometHashAggregate +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#18] -Results [2]: [substr(ca_zip#13, 1, 5) AS ca_zip#19, count(1)#18 AS cnt#20] -(34) Filter [codegen id : 5] +(30) CometFilter Input [2]: [ca_zip#19, cnt#20] Condition : (cnt#20 > 10) -(35) Project [codegen id : 5] -Output [1]: [ca_zip#19] +(31) CometProject Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] -(36) BroadcastExchange +(32) CometBroadcastExchange Input [1]: [ca_zip#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=5] +Arguments: [ca_zip#19] -(37) BroadcastHashJoin [codegen id : 6] -Left keys [2]: [coalesce(ca_zip#11, ), isnull(ca_zip#11)] -Right keys [2]: [coalesce(ca_zip#19, ), isnull(ca_zip#19)] -Join type: LeftSemi -Join condition: None +(33) CometBroadcastHashJoin +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight -(38) HashAggregate [codegen id : 6] -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(34) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -Aggregate Attributes: [] -Results [1]: [ca_zip#11] -(39) Exchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(35) CometExchange +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(40) HashAggregate [codegen id : 7] -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(36) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -Aggregate Attributes: [] -Results [1]: [ca_zip#11] -(41) BroadcastExchange -Input [1]: [ca_zip#11] -Arguments: HashedRelationBroadcastMode(List(substr(input[0, string, true], 1, 2)),false), [plan_id=7] +(37) CometBroadcastExchange +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [substr(s_zip#9, 1, 2)] -Right keys [1]: [substr(ca_zip#11, 1, 2)] -Join type: Inner -Join condition: None +(38) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight -(43) Project [codegen id : 8] -Output [2]: [ss_net_profit#2, s_store_name#8] -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] +(39) CometProject +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] +Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] -(44) HashAggregate [codegen id : 8] +(40) CometHashAggregate Input [2]: [ss_net_profit#2, s_store_name#8] Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#21] -Results [2]: [s_store_name#8, sum#22] -(45) Exchange -Input [2]: [s_store_name#8, sum#22] -Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(41) CometExchange +Input [2]: [s_store_name#8, sum#21] +Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(46) HashAggregate [codegen id : 9] -Input [2]: [s_store_name#8, sum#22] +(42) CometHashAggregate +Input [2]: [s_store_name#8, sum#21] Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] -Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS sum(ss_net_profit)#24] -(47) TakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#24] -Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#24] +(43) CometTakeOrderedAndProject +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#22]), [s_store_name#8, sum(ss_net_profit)#22], 100, 0, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] + +(44) CometColumnarToRow [codegen id : 1] +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/extended.txt new file mode 100644 index 0000000000..c84e490026 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/extended.txt @@ -0,0 +1,46 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 43 out of 43 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/simplified.txt index 30895d0bef..123d33addd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/simplified.txt @@ -1,70 +1,46 @@ -TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - WholeStageCodegen (9) - HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum] - InputAdapter - Exchange [s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [s_store_name,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_store_name] - BroadcastHashJoin [s_zip,ca_zip] - Project [ss_net_profit,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [ca_zip] - InputAdapter - Exchange [ca_zip] #5 - WholeStageCodegen (6) - HashAggregate [ca_zip] - BroadcastHashJoin [ca_zip,ca_zip] - Project [ca_zip] - Filter [ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_zip] - Filter [cnt] - HashAggregate [ca_zip,count] [count(1),ca_zip,cnt,count] - InputAdapter - Exchange [ca_zip] #7 - WholeStageCodegen (4) - HashAggregate [ca_zip] [count,count] - Project [ca_zip] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (3) - Project [c_current_addr_sk] - Filter [c_preferred_cust_flag,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,sum(ss_net_profit)] + CometHashAggregate [sum] [s_store_name,sum(ss_net_profit),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_name] #1 + CometHashAggregate [ss_net_profit] [s_store_name,sum] + CometProject [ss_net_profit,s_store_name] + CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] + CometProject [ss_net_profit,s_store_name,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [ca_zip] #4 + CometHashAggregate [ca_zip] + CometExchange [ca_zip] #5 + CometHashAggregate [ca_zip] + CometBroadcastHashJoin [ca_zip,ca_zip] + CometProject [ca_zip] [ca_zip] + CometFilter [ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_zip] + CometBroadcastExchange [ca_zip] #6 + CometProject [ca_zip] + CometFilter [ca_zip,cnt] + CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] + CometExchange [ca_zip] #7 + CometHashAggregate [ca_zip,count] + CometProject [ca_zip] + CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [c_current_addr_sk] #8 + CometProject [c_current_addr_sk] + CometFilter [c_current_addr_sk,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/explain.txt index fd964a80ef..dda2ae6aa9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/explain.txt @@ -1,631 +1,574 @@ == Physical Plan == -TakeOrderedAndProject (111) -+- * HashAggregate (110) - +- Exchange (109) - +- * HashAggregate (108) - +- * Expand (107) - +- Union (106) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Project (20) - : : : : +- * BroadcastHashJoin Inner BuildRight (19) - : : : : :- * Project (13) - : : : : : +- * SortMergeJoin LeftOuter (12) - : : : : : :- * Sort (5) - : : : : : : +- Exchange (4) - : : : : : : +- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- * Sort (11) - : : : : : +- Exchange (10) - : : : : : +- * Project (9) - : : : : : +- * Filter (8) - : : : : : +- * ColumnarToRow (7) - : : : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : : : +- BroadcastExchange (18) - : : : : +- * Project (17) - : : : : +- * Filter (16) - : : : : +- * ColumnarToRow (15) - : : : : +- Scan parquet spark_catalog.default.date_dim (14) - : : : +- BroadcastExchange (24) - : : : +- * Filter (23) - : : : +- * ColumnarToRow (22) - : : : +- Scan parquet spark_catalog.default.store (21) - : : +- BroadcastExchange (31) - : : +- * Project (30) - : : +- * Filter (29) - : : +- * ColumnarToRow (28) - : : +- Scan parquet spark_catalog.default.item (27) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.promotion (34) - :- * HashAggregate (74) - : +- Exchange (73) - : +- * HashAggregate (72) - : +- * Project (71) - : +- * BroadcastHashJoin Inner BuildRight (70) - : :- * Project (68) - : : +- * BroadcastHashJoin Inner BuildRight (67) - : : :- * Project (65) - : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : :- * Project (59) - : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : :- * Project (56) - : : : : : +- * SortMergeJoin LeftOuter (55) - : : : : : :- * Sort (48) - : : : : : : +- Exchange (47) - : : : : : : +- * Filter (46) - : : : : : : +- * ColumnarToRow (45) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (44) - : : : : : +- * Sort (54) - : : : : : +- Exchange (53) - : : : : : +- * Project (52) - : : : : : +- * Filter (51) - : : : : : +- * ColumnarToRow (50) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (49) - : : : : +- ReusedExchange (57) - : : : +- BroadcastExchange (63) - : : : +- * Filter (62) - : : : +- * ColumnarToRow (61) - : : : +- Scan parquet spark_catalog.default.catalog_page (60) - : : +- ReusedExchange (66) - : +- ReusedExchange (69) - +- * HashAggregate (105) - +- Exchange (104) - +- * HashAggregate (103) - +- * Project (102) - +- * BroadcastHashJoin Inner BuildRight (101) - :- * Project (99) - : +- * BroadcastHashJoin Inner BuildRight (98) - : :- * Project (96) - : : +- * BroadcastHashJoin Inner BuildRight (95) - : : :- * Project (90) - : : : +- * BroadcastHashJoin Inner BuildRight (89) - : : : :- * Project (87) - : : : : +- * SortMergeJoin LeftOuter (86) - : : : : :- * Sort (79) - : : : : : +- Exchange (78) - : : : : : +- * Filter (77) - : : : : : +- * ColumnarToRow (76) - : : : : : +- Scan parquet spark_catalog.default.web_sales (75) - : : : : +- * Sort (85) - : : : : +- Exchange (84) - : : : : +- * Project (83) - : : : : +- * Filter (82) - : : : : +- * ColumnarToRow (81) - : : : : +- Scan parquet spark_catalog.default.web_returns (80) - : : : +- ReusedExchange (88) - : : +- BroadcastExchange (94) - : : +- * Filter (93) - : : +- * ColumnarToRow (92) - : : +- Scan parquet spark_catalog.default.web_site (91) - : +- ReusedExchange (97) - +- ReusedExchange (100) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (103) ++- CometTakeOrderedAndProject (102) + +- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometExpand (98) + +- CometUnion (97) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (17) + : : : : +- CometBroadcastHashJoin (16) + : : : : :- CometProject (11) + : : : : : +- CometSortMergeJoin (10) + : : : : : :- CometSort (4) + : : : : : : +- CometExchange (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometSort (9) + : : : : : +- CometExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : : : +- CometBroadcastExchange (15) + : : : : +- CometProject (14) + : : : : +- CometFilter (13) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : : : +- CometBroadcastExchange (21) + : : : +- CometProject (20) + : : : +- CometFilter (19) + : : : +- CometNativeScan parquet spark_catalog.default.store (18) + : : +- CometBroadcastExchange (27) + : : +- CometProject (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.item (24) + : +- CometBroadcastExchange (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometNativeScan parquet spark_catalog.default.promotion (30) + :- CometHashAggregate (67) + : +- CometExchange (66) + : +- CometHashAggregate (65) + : +- CometProject (64) + : +- CometBroadcastHashJoin (63) + : :- CometProject (61) + : : +- CometBroadcastHashJoin (60) + : : :- CometProject (58) + : : : +- CometBroadcastHashJoin (57) + : : : :- CometProject (52) + : : : : +- CometBroadcastHashJoin (51) + : : : : :- CometProject (49) + : : : : : +- CometSortMergeJoin (48) + : : : : : :- CometSort (42) + : : : : : : +- CometExchange (41) + : : : : : : +- CometFilter (40) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (39) + : : : : : +- CometSort (47) + : : : : : +- CometExchange (46) + : : : : : +- CometProject (45) + : : : : : +- CometFilter (44) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (43) + : : : : +- ReusedExchange (50) + : : : +- CometBroadcastExchange (56) + : : : +- CometProject (55) + : : : +- CometFilter (54) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (53) + : : +- ReusedExchange (59) + : +- ReusedExchange (62) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometBroadcastHashJoin (86) + : : :- CometProject (81) + : : : +- CometBroadcastHashJoin (80) + : : : :- CometProject (78) + : : : : +- CometSortMergeJoin (77) + : : : : :- CometSort (71) + : : : : : +- CometExchange (70) + : : : : : +- CometFilter (69) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (68) + : : : : +- CometSort (76) + : : : : +- CometExchange (75) + : : : : +- CometProject (74) + : : : : +- CometFilter (73) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (72) + : : : +- ReusedExchange (79) + : : +- CometBroadcastExchange (85) + : : +- CometProject (84) + : : +- CometFilter (83) + : : +- CometNativeScan parquet spark_catalog.default.web_site (82) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +(7) CometProject Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -(10) Exchange +(8) CometExchange Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] -Right keys [2]: [sr_item_sk#8, sr_ticket_number#9] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter -(13) Project [codegen id : 9] -Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +(11) CometProject Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(14) Scan parquet spark_catalog.default.date_dim +(12) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] - -(16) Filter [codegen id : 5] +(13) CometFilter Input [2]: [d_date_sk#13, d_date#14] Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) -(17) Project [codegen id : 5] -Output [1]: [d_date_sk#13] +(14) CometProject Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(18) BroadcastExchange +(15) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#13] -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight -(20) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +(17) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(21) Scan parquet spark_catalog.default.store +(18) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] - -(23) Filter [codegen id : 6] +(19) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(24) BroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] -(26) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] +(22) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(27) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] - -(29) Filter [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) -(30) Project [codegen id : 7] -Output [1]: [i_item_sk#17] -Input [2]: [i_item_sk#17, i_current_price#18] +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] -(31) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(33) Project [codegen id : 9] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(34) Scan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] +(30) CometNativeScan parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) -(36) Filter [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] -(37) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] -Input [2]: [p_promo_sk#19, p_channel_tv#20] +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] -(38) BroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(40) Project [codegen id : 9] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] - -(41) HashAggregate [codegen id : 9] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -(42) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(43) HashAggregate [codegen id : 10] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] +(38) CometHashAggregate +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#34, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#35, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#36, store channel AS channel#37, concat(store, s_store_id#16) AS id#38] -(44) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(39) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(46) Filter [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) +(40) CometFilter +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : ((isnotnull(cs_catalog_page_sk#27) AND isnotnull(cs_item_sk#28)) AND isnotnull(cs_promo_sk#29)) -(47) Exchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(41) CometExchange +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: hashpartitioning(cs_item_sk#28, cs_order_number#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(48) Sort [codegen id : 12] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST], false, 0 +(42) CometSort +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33], [cs_item_sk#28 ASC NULLS FIRST, cs_order_number#30 ASC NULLS FIRST] -(49) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(43) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(44) CometFilter +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Condition : (isnotnull(cr_item_sk#34) AND isnotnull(cr_order_number#35)) -(51) Filter [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Condition : (isnotnull(cr_item_sk#46) AND isnotnull(cr_order_number#47)) +(45) CometProject +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] -(52) Project [codegen id : 13] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(46) CometExchange +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: hashpartitioning(cr_item_sk#34, cr_order_number#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(53) Exchange -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: hashpartitioning(cr_item_sk#46, cr_order_number#47, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(47) CometSort +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34 ASC NULLS FIRST, cr_order_number#35 ASC NULLS FIRST] -(54) Sort [codegen id : 14] -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST], false, 0 +(48) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_item_sk#28, cs_order_number#30], [cr_item_sk#34, cr_order_number#35], LeftOuter -(55) SortMergeJoin [codegen id : 19] -Left keys [2]: [cs_item_sk#40, cs_order_number#42] -Right keys [2]: [cr_item_sk#46, cr_order_number#47] -Join type: LeftOuter -Join condition: None +(49) CometProject +Input [11]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] -(56) Project [codegen id : 19] -Output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] +(50) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#39] -(57) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#51] +(51) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] +Right output [1]: [d_date_sk#39] +Arguments: [cs_sold_date_sk#33], [d_date_sk#39], Inner, BuildRight -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None +(52) CometProject +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37, d_date_sk#39] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] -(59) Project [codegen id : 19] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#51] - -(60) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] +(53) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(62) Filter [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Condition : isnotnull(cp_catalog_page_sk#52) - -(63) BroadcastExchange -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(64) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#52] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 19] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(66) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#54] - -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#54] -Join type: Inner -Join condition: None - -(68) Project [codegen id : 19] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, i_item_sk#54] - -(69) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#55] - -(70) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(71) Project [codegen id : 19] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, p_promo_sk#55] - -(72) HashAggregate [codegen id : 19] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Results [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] - -(73) Exchange -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) HashAggregate [codegen id : 20] -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#66, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#66,17,2) AS sales#69, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67 AS returns#70, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68 AS profit#71, catalog channel AS channel#72, concat(catalog_page, cp_catalog_page_id#53) AS id#73] - -(75) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] +(54) CometFilter +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Condition : isnotnull(cp_catalog_page_sk#40) + +(55) CometProject +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42], [cp_catalog_page_sk#40, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#41, 16, true, false, true) AS cp_catalog_page_id#42] + +(56) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42] + +(57) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] +Right output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_catalog_page_sk#27], [cp_catalog_page_sk#40], Inner, BuildRight + +(58) CometProject +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(59) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#43] + +(60) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [i_item_sk#43] +Arguments: [cs_item_sk#28], [i_item_sk#43], Inner, BuildRight + +(61) CometProject +Input [8]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, i_item_sk#43] +Arguments: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(62) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#44] + +(63) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [p_promo_sk#44] +Arguments: [cs_promo_sk#29], [p_promo_sk#44], Inner, BuildRight + +(64) CometProject +Input [7]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, p_promo_sk#44] +Arguments: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(65) CometHashAggregate +Input [5]: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(66) CometExchange +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Arguments: hashpartitioning(cp_catalog_page_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(67) CometHashAggregate +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(68) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#80)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#56)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] - -(77) Filter [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Condition : ((isnotnull(ws_web_site_sk#75) AND isnotnull(ws_item_sk#74)) AND isnotnull(ws_promo_sk#76)) +(69) CometFilter +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Condition : ((isnotnull(ws_web_site_sk#51) AND isnotnull(ws_item_sk#50)) AND isnotnull(ws_promo_sk#52)) -(78) Exchange -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: hashpartitioning(ws_item_sk#74, ws_order_number#77, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(70) CometExchange +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: hashpartitioning(ws_item_sk#50, ws_order_number#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(79) Sort [codegen id : 22] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: [ws_item_sk#74 ASC NULLS FIRST, ws_order_number#77 ASC NULLS FIRST], false, 0 +(71) CometSort +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56], [ws_item_sk#50 ASC NULLS FIRST, ws_order_number#53 ASC NULLS FIRST] -(80) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(72) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(81) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] - -(82) Filter [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Condition : (isnotnull(wr_item_sk#81) AND isnotnull(wr_order_number#82)) +(73) CometFilter +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Condition : (isnotnull(wr_item_sk#57) AND isnotnull(wr_order_number#58)) -(83) Project [codegen id : 23] -Output [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(74) CometProject +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] -(84) Exchange -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: hashpartitioning(wr_item_sk#81, wr_order_number#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(75) CometExchange +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: hashpartitioning(wr_item_sk#57, wr_order_number#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(85) Sort [codegen id : 24] -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: [wr_item_sk#81 ASC NULLS FIRST, wr_order_number#82 ASC NULLS FIRST], false, 0 +(76) CometSort +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57 ASC NULLS FIRST, wr_order_number#58 ASC NULLS FIRST] -(86) SortMergeJoin [codegen id : 29] -Left keys [2]: [ws_item_sk#74, ws_order_number#77] -Right keys [2]: [wr_item_sk#81, wr_order_number#82] -Join type: LeftOuter -Join condition: None +(77) CometSortMergeJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Right output [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_order_number#53], [wr_item_sk#57, wr_order_number#58], LeftOuter -(87) Project [codegen id : 29] -Output [8]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84] -Input [11]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] +(78) CometProject +Input [11]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] -(88) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#86] +(79) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#62] -(89) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_sold_date_sk#80] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None +(80) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] +Right output [1]: [d_date_sk#62] +Arguments: [ws_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight -(90) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84, d_date_sk#86] +(81) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60, d_date_sk#62] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] -(91) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#87, web_site_id#88] +(82) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#63, web_site_id#64] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(92) ColumnarToRow [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] - -(93) Filter [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] -Condition : isnotnull(web_site_sk#87) - -(94) BroadcastExchange -Input [2]: [web_site_sk#87, web_site_id#88] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_web_site_sk#75] -Right keys [1]: [web_site_sk#87] -Join type: Inner -Join condition: None - -(96) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_sk#87, web_site_id#88] - -(97) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#89] - -(98) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_item_sk#74] -Right keys [1]: [i_item_sk#89] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 29] -Output [6]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [8]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, i_item_sk#89] - -(100) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#90] - -(101) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_promo_sk#76] -Right keys [1]: [p_promo_sk#90] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 29] -Output [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [7]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, p_promo_sk#90] - -(103) HashAggregate [codegen id : 29] -Input [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Keys [1]: [web_site_id#88] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#78)), partial_sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Results [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] - -(104) Exchange -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(web_site_id#88, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(105) HashAggregate [codegen id : 30] -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Keys [1]: [web_site_id#88] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#78)), sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#78))#101, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#78))#101,17,2) AS sales#104, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102 AS returns#105, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103 AS profit#106, web channel AS channel#107, concat(web_site, web_site_id#88) AS id#108] - -(106) Union - -(107) Expand [codegen id : 31] -Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] -Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#109, id#110, spark_grouping_id#111] - -(108) HashAggregate [codegen id : 31] -Input [6]: [sales#34, returns#35, profit#36, channel#109, id#110, spark_grouping_id#111] -Keys [3]: [channel#109, id#110, spark_grouping_id#111] -Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] -Aggregate Attributes [6]: [sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Results [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] - -(109) Exchange -Input [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Arguments: hashpartitioning(channel#109, id#110, spark_grouping_id#111, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(110) HashAggregate [codegen id : 32] -Input [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Keys [3]: [channel#109, id#110, spark_grouping_id#111] -Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] -Aggregate Attributes [3]: [sum(sales#34)#124, sum(returns#35)#125, sum(profit#36)#126] -Results [5]: [channel#109, id#110, sum(sales#34)#124 AS sales#127, sum(returns#35)#125 AS returns#128, sum(profit#36)#126 AS profit#129] - -(111) TakeOrderedAndProject -Input [5]: [channel#109, id#110, sales#127, returns#128, profit#129] -Arguments: 100, [channel#109 ASC NULLS FIRST, id#110 ASC NULLS FIRST], [channel#109, id#110, sales#127, returns#128, profit#129] +(83) CometFilter +Input [2]: [web_site_sk#63, web_site_id#64] +Condition : isnotnull(web_site_sk#63) + +(84) CometProject +Input [2]: [web_site_sk#63, web_site_id#64] +Arguments: [web_site_sk#63, web_site_id#65], [web_site_sk#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#64, 16, true, false, true) AS web_site_id#65] + +(85) CometBroadcastExchange +Input [2]: [web_site_sk#63, web_site_id#65] +Arguments: [web_site_sk#63, web_site_id#65] + +(86) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] +Right output [2]: [web_site_sk#63, web_site_id#65] +Arguments: [ws_web_site_sk#51], [web_site_sk#63], Inner, BuildRight + +(87) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_sk#63, web_site_id#65] +Arguments: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(88) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#66] + +(89) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [i_item_sk#66] +Arguments: [ws_item_sk#50], [i_item_sk#66], Inner, BuildRight + +(90) CometProject +Input [8]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, i_item_sk#66] +Arguments: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(91) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#67] + +(92) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [p_promo_sk#67] +Arguments: [ws_promo_sk#52], [p_promo_sk#67], Inner, BuildRight + +(93) CometProject +Input [7]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, p_promo_sk#67] +Arguments: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(94) CometHashAggregate +Input [5]: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Keys [1]: [web_site_id#65] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#54)), partial_sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(95) CometExchange +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Arguments: hashpartitioning(web_site_id#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(96) CometHashAggregate +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Keys [1]: [web_site_id#65] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#54)), sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(97) CometUnion +Child 0 Input [5]: [sales#73, returns#74, profit#75, channel#76, id#77] +Child 1 Input [5]: [sales#78, returns#79, profit#80, channel#81, id#82] +Child 2 Input [5]: [sales#83, returns#84, profit#85, channel#86, id#87] + +(98) CometExpand +Input [5]: [sales#73, returns#74, profit#75, channel#76, id#77] +Arguments: [[sales#73, returns#74, profit#75, channel#76, id#77, 0], [sales#73, returns#74, profit#75, channel#76, null, 1], [sales#73, returns#74, profit#75, null, null, 3]], [sales#73, returns#74, profit#75, channel#88, id#89, spark_grouping_id#90] + +(99) CometHashAggregate +Input [6]: [sales#73, returns#74, profit#75, channel#88, id#89, spark_grouping_id#90] +Keys [3]: [channel#88, id#89, spark_grouping_id#90] +Functions [3]: [partial_sum(sales#73), partial_sum(returns#74), partial_sum(profit#75)] + +(100) CometExchange +Input [9]: [channel#88, id#89, spark_grouping_id#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(channel#88, id#89, spark_grouping_id#90, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(101) CometHashAggregate +Input [9]: [channel#88, id#89, spark_grouping_id#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Keys [3]: [channel#88, id#89, spark_grouping_id#90] +Functions [3]: [sum(sales#73), sum(returns#74), sum(profit#75)] + +(102) CometTakeOrderedAndProject +Input [5]: [channel#88, id#89, sales#97, returns#98, profit#99] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#88 ASC NULLS FIRST,id#89 ASC NULLS FIRST], output=[channel#88,id#89,sales#97,returns#98,profit#99]), [channel#88, id#89, sales#97, returns#98, profit#99], 100, 0, [channel#88 ASC NULLS FIRST, id#89 ASC NULLS FIRST], [channel#88, id#89, sales#97, returns#98, profit#99] + +(103) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#88, id#89, sales#97, returns#98, profit#99] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/extended.txt new file mode 100644 index 0000000000..dd8065cb77 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/extended.txt @@ -0,0 +1,123 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_site + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 120 out of 120 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/simplified.txt index 3b9ea7c700..4a2e5b04ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/simplified.txt @@ -1,178 +1,105 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (32) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (31) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (10) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (9) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #3 - WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (20) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #9 - WholeStageCodegen (19) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (12) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #10 - WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (14) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #11 - WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - InputAdapter - ReusedExchange [i_item_sk] #7 - InputAdapter - ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (30) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #13 - WholeStageCodegen (29) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - WholeStageCodegen (22) - Sort [ws_item_sk,ws_order_number] - InputAdapter - Exchange [ws_item_sk,ws_order_number] #14 - WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (24) - Sort [wr_item_sk,wr_order_number] - InputAdapter - Exchange [wr_item_sk,wr_order_number] #15 - WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - InputAdapter - ReusedExchange [i_item_sk] #7 - InputAdapter - ReusedExchange [p_promo_sk] #8 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id,spark_grouping_id] #1 + CometHashAggregate [sales,returns,profit] [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometExpand [channel,id] [sales,returns,profit,channel,id,spark_grouping_id] + CometUnion [sales,returns,profit,channel,id] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),s_store_id] + CometExchange [s_store_id] #2 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #4 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #8 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),cp_catalog_page_id] + CometExchange [cp_catalog_page_id] #9 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #10 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #11 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [sales,returns,profit,channel,id,sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),web_site_id] + CometExchange [web_site_id] #13 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #14 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #15 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/explain.txt index 829e66974d..20eef2cab9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/explain.txt @@ -1,307 +1,283 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * Project (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_address (11) - : : +- BroadcastExchange (37) - : : +- * Filter (36) - : : +- * HashAggregate (35) - : : +- Exchange (34) - : : +- * HashAggregate (33) - : : +- * HashAggregate (32) - : : +- Exchange (31) - : : +- * HashAggregate (30) - : : +- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.catalog_returns (21) - : : : +- ReusedExchange (24) - : : +- ReusedExchange (27) - : +- BroadcastExchange (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer (40) - +- BroadcastExchange (49) - +- * Filter (48) - +- * ColumnarToRow (47) - +- Scan parquet spark_catalog.default.customer_address (46) - - -(1) Scan parquet spark_catalog.default.catalog_returns +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometNativeScan parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometNativeScan parquet spark_catalog.default.customer_address (43) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_year#6] Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +(8) CometProject Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] +Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(13) Filter [codegen id : 2] +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(14) BroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] +(14) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] -(17) HashAggregate [codegen id : 3] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -(18) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 11] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] -(20) Filter [codegen id : 11] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(21) Scan parquet spark_catalog.default.catalog_returns +(19) CometNativeScan parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(23) Filter [codegen id : 6] +(20) CometFilter Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Condition : isnotnull(cr_returning_addr_sk#2) -(24) ReusedExchange [Reuses operator id: 8] +(21) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#5] -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(26) Project [codegen id : 6] -Output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +(23) CometProject Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] +Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -(27) ReusedExchange [Reuses operator id: 14] -Output [2]: [ca_address_sk#7, ca_state#8] +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#7, ca_state#9] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(29) Project [codegen id : 6] -Output [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] +(26) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] -(30) HashAggregate [codegen id : 6] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(27) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#15] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -(31) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#14] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(32) HashAggregate [codegen id : 7] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(29) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#14] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [2]: [ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] - -(33) HashAggregate [codegen id : 7] -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [ctr_state#13, sum#19, count#20] - -(34) Exchange -Input [3]: [ctr_state#13, sum#19, count#20] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 8] -Input [3]: [ctr_state#13, sum#19, count#20] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] -Aggregate Attributes [1]: [avg(ctr_total_return#14)#21] -Results [2]: [(avg(ctr_total_return#14)#21 * 1.2) AS (avg(ctr_total_return) * 1.2)#22, ctr_state#13 AS ctr_state#13#23] - -(36) Filter [codegen id : 8] -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#22) - -(37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#13#23] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#22) - -(39) Project [codegen id : 11] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] - -(40) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] + +(30) CometHashAggregate +Input [2]: [ctr_state#12, ctr_total_return#13] +Keys [1]: [ctr_state#12] +Functions [1]: [partial_avg(ctr_total_return#13)] + +(31) CometExchange +Input [3]: [ctr_state#12, sum#15, count#16] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(32) CometHashAggregate +Input [3]: [ctr_state#12, sum#15, count#16] +Keys [1]: [ctr_state#12] +Functions [1]: [avg(ctr_total_return#13)] + +(33) CometFilter +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#17) + +(34) CometBroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] + +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_state#12], [ctr_state#12#18], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#17), BuildRight + +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] + +(37) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] +(38) CometFilter +Input [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#21)) -(42) Filter [codegen id : 9] -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#26)) +(39) CometProject +Input [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28], [c_customer_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#20, 16, true, false, true) AS c_customer_id#25, c_current_addr_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#22, 10, true, false, true) AS c_salutation#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#28] -(43) BroadcastExchange -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(40) CometBroadcastExchange +Input [6]: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] -(44) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [6]: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [ctr_customer_sk#11], [c_customer_sk#19], Inner, BuildRight -(45) Project [codegen id : 11] -Output [6]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] +(42) CometProject +Input [8]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28], [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] -(46) Scan parquet spark_catalog.default.customer_address -Output [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] +(43) CometNativeScan parquet spark_catalog.default.customer_address +Output [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 10] -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] +(44) CometFilter +Input [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#36, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#29)) + +(45) CometProject +Input [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] +Arguments: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46], [ca_address_sk#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#30, 10, true, false, true) AS ca_street_number#41, ca_street_name#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#32, 15, true, false, true) AS ca_street_type#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#33, 10, true, false, true) AS ca_suite_number#43, ca_city#34, ca_county#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#36, 2, true, false, true) AS ca_state#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#37, 10, true, false, true) AS ca_zip#45, ca_country#38, ca_gmt_offset#39, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#40, 20, true, false, true) AS ca_location_type#46] -(48) Filter [codegen id : 10] -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] -Condition : ((isnotnull(ca_state#37) AND (ca_state#37 = GA)) AND isnotnull(ca_address_sk#30)) +(46) CometBroadcastExchange +Input [12]: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] -(49) BroadcastExchange -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(47) CometBroadcastHashJoin +Left output [6]: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Right output [12]: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [c_current_addr_sk#21], [ca_address_sk#29], Inner, BuildRight -(50) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#26] -Right keys [1]: [ca_address_sk#30] -Join type: Inner -Join condition: None +(48) CometProject +Input [18]: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28, ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13], [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] -(51) Project [codegen id : 11] -Output [16]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] -Input [18]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] +(49) CometTakeOrderedAndProject +Input [16]: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#25 ASC NULLS FIRST,c_salutation#26 ASC NULLS FIRST,c_first_name#27 ASC NULLS FIRST,c_last_name#28 ASC NULLS FIRST,ca_street_number#41 ASC NULLS FIRST,ca_street_name#31 ASC NULLS FIRST,ca_street_type#42 ASC NULLS FIRST,ca_suite_number#43 ASC NULLS FIRST,ca_city#34 ASC NULLS FIRST,ca_county#35 ASC NULLS FIRST,ca_state#44 ASC NULLS FIRST,ca_zip#45 ASC NULLS FIRST,ca_country#38 ASC NULLS FIRST,ca_gmt_offset#39 ASC NULLS FIRST,ca_location_type#46 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#25,c_salutation#26,c_first_name#27,c_last_name#28,ca_street_number#41,ca_street_name#31,ca_street_type#42,ca_suite_number#43,ca_city#34,ca_county#35,ca_state#44,ca_zip#45,ca_country#38,ca_gmt_offset#39,ca_location_type#46,ctr_total_return#13]), [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13], 100, 0, [c_customer_id#25 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_last_name#28 ASC NULLS FIRST, ca_street_number#41 ASC NULLS FIRST, ca_street_name#31 ASC NULLS FIRST, ca_street_type#42 ASC NULLS FIRST, ca_suite_number#43 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, ca_county#35 ASC NULLS FIRST, ca_state#44 ASC NULLS FIRST, ca_zip#45 ASC NULLS FIRST, ca_country#38 ASC NULLS FIRST, ca_gmt_offset#39 ASC NULLS FIRST, ca_location_type#46 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] -(52) TakeOrderedAndProject -Input [16]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST, c_salutation#27 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, c_last_name#29 ASC NULLS FIRST, ca_street_number#31 ASC NULLS FIRST, ca_street_name#32 ASC NULLS FIRST, ca_street_type#33 ASC NULLS FIRST, ca_suite_number#34 ASC NULLS FIRST, ca_city#35 ASC NULLS FIRST, ca_county#36 ASC NULLS FIRST, ca_state#37 ASC NULLS FIRST, ca_zip#38 ASC NULLS FIRST, ca_country#39 ASC NULLS FIRST, ca_gmt_offset#40 ASC NULLS FIRST, ca_location_type#41 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] +(50) CometColumnarToRow [codegen id : 1] +Input [16]: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/extended.txt new file mode 100644 index 0000000000..188bc4f72b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/extended.txt @@ -0,0 +1,58 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 55 out of 55 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/simplified.txt index 2defde832f..904f67113d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/simplified.txt @@ -1,77 +1,52 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - WholeStageCodegen (11) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (3) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] - Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (8) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (7) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (6) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] - Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] + CometBroadcastHashJoin [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometProject [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometBroadcastHashJoin [ctr_customer_sk,ctr_total_return,c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometProject [ctr_customer_sk,ctr_total_return] + CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] + CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometExchange [cr_returning_customer_sk,ca_state] #1 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 + CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return),ctr_state] + CometExchange [ctr_state] #5 + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [cr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometExchange [cr_returning_customer_sk,ca_state] #6 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #7 + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #8 + CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/explain.txt index 67550028f1..f63414fbe4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/explain.txt @@ -1,169 +1,150 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildLeft (24) - :- BroadcastExchange (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.inventory (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) - +- Scan parquet spark_catalog.default.store_sales (20) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (26) ++- CometTakeOrderedAndProject (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.inventory (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + +- CometProject (19) + +- CometFilter (18) + +- CometNativeScan parquet spark_catalog.default.store_sales (17) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,423,821]), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +(3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(5) Scan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +(5) CometFilter +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(6) CometProject +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] -(9) BroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(7) CometBroadcastExchange +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [inv_item_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight -(11) Project [codegen id : 3] -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] +(9) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(12) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] +(10) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] +(11) CometFilter +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) -(14) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) +(12) CometProject +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] +(13) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(16) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(14) CometBroadcastHashJoin +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [inv_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(18) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] +(16) CometBroadcastExchange +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(19) BroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] +(17) CometNativeScan parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(21) ColumnarToRow -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] - -(22) Filter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) +(18) CometFilter +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) -(23) Project -Output [1]: [ss_item_sk#11] -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] +(19) CometProject +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#11] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft -(25) Project [codegen id : 4] -Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] +(21) CometProject +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) HashAggregate [codegen id : 4] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(22) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(27) Exchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(24) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(29) TakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +(25) CometTakeOrderedAndProject +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/extended.txt new file mode 100644 index 0000000000..6e00d25e4c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.inventory + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_sales + +Comet accelerated 25 out of 25 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/simplified.txt index 1ab87aee6a..4deaf58d7e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/simplified.txt @@ -1,42 +1,28 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - WholeStageCodegen (5) - HashAggregate [i_item_id,i_item_desc,i_current_price] - InputAdapter - Exchange [i_item_id,i_item_desc,i_current_price] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_current_price] - Project [i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [i_item_sk,ss_item_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (3) - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - Project [ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometExchange [i_item_id,i_item_desc,i_current_price] #1 + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [ss_item_sk] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/explain.txt index 2a4ba79bf8..ec24977bc2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/explain.txt @@ -1,357 +1,368 @@ == Physical Plan == -TakeOrderedAndProject (61) -+- * Project (60) - +- * BroadcastHashJoin Inner BuildRight (59) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_returns (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin LeftSemi BuildRight (23) - : : :- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin LeftSemi BuildRight (20) - : : :- * ColumnarToRow (14) - : : : +- Scan parquet spark_catalog.default.date_dim (13) - : : +- BroadcastExchange (19) - : : +- * Project (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- BroadcastExchange (43) - : +- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Filter (33) - : : : +- * ColumnarToRow (32) - : : : +- Scan parquet spark_catalog.default.catalog_returns (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- BroadcastExchange (58) - +- * HashAggregate (57) - +- Exchange (56) - +- * HashAggregate (55) - +- * Project (54) - +- * BroadcastHashJoin Inner BuildRight (53) - :- * Project (51) - : +- * BroadcastHashJoin Inner BuildRight (50) - : :- * Filter (48) - : : +- * ColumnarToRow (47) - : : +- Scan parquet spark_catalog.default.web_returns (46) - : +- ReusedExchange (49) - +- ReusedExchange (52) - - -(1) Scan parquet spark_catalog.default.store_returns +* CometColumnarToRow (65) ++- CometTakeOrderedAndProject (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (49) + : +- CometBroadcastHashJoin (48) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometNativeScan parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (47) + : +- CometHashAggregate (46) + : +- CometExchange (45) + : +- CometHashAggregate (44) + : +- CometProject (43) + : +- CometBroadcastHashJoin (42) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- CometBroadcastExchange (41) + : +- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometFilter (33) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (32) + : +- CometBroadcastExchange (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometNativeScan parquet spark_catalog.default.date_dim (34) + : +- ReusedExchange (35) + +- CometBroadcastExchange (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometProject (57) + +- CometBroadcastHashJoin (56) + :- CometProject (54) + : +- CometBroadcastHashJoin (53) + : :- CometFilter (51) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (50) + : +- ReusedExchange (52) + +- ReusedExchange (55) + + +(1) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#3)] PushedFilters: [IsNotNull(sr_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(6) Filter [codegen id : 1] +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(7) BroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 5] -Output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] +(8) CometProject +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#6, d_date#7] - -(12) Filter [codegen id : 4] -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(13) Scan parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [d_date#8, d_week_seq#9] - -(15) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#11] +(12) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(16) ColumnarToRow [codegen id : 2] -Input [2]: [d_date#10, d_week_seq#11] - -(17) Filter [codegen id : 2] -Input [2]: [d_date#10, d_week_seq#11] -Condition : cast(d_date#10 as string) IN (2000-06-30,2000-09-27,2000-11-17) +(13) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : cast(d_date#8 as string) IN (2000-06-30,2000-09-27,2000-11-17) -(18) Project [codegen id : 2] -Output [1]: [d_week_seq#11] -Input [2]: [d_date#10, d_week_seq#11] +(14) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_week_seq#9#10], [d_week_seq#9 AS d_week_seq#9#10] -(19) BroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(15) CometBroadcastExchange +Input [1]: [d_week_seq#9#10] +Arguments: [d_week_seq#9#10] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_week_seq#9] -Right keys [1]: [d_week_seq#11] -Join type: LeftSemi -Join condition: None +(16) CometBroadcastHashJoin +Left output [2]: [d_date#8, d_week_seq#9] +Right output [1]: [d_week_seq#9#10] +Arguments: [d_week_seq#9], [d_week_seq#9#10], LeftSemi, BuildRight -(21) Project [codegen id : 3] -Output [1]: [d_date#8] +(17) CometProject Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8#11], [d_date#8 AS d_date#8#11] -(22) BroadcastExchange -Input [1]: [d_date#8] -Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [d_date#7] -Right keys [1]: [d_date#8] -Join type: LeftSemi -Join condition: None - -(24) Project [codegen id : 4] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_date#7] - -(25) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_returned_date_sk#3] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 5] -Output [2]: [sr_return_quantity#2, i_item_id#5] -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] - -(28) HashAggregate [codegen id : 5] -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(18) CometBroadcastExchange +Input [1]: [d_date#8#11] +Arguments: [d_date#8#11] + +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#8#11] +Arguments: [d_date#8], [d_date#8#11], LeftSemi, BuildRight + +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] -(29) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometExchange +Input [2]: [i_item_id#6, sum#12] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(30) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#12] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum(sr_return_quantity#2)#14] -Results [2]: [i_item_id#5 AS item_id#15, sum(sr_return_quantity#2)#14 AS sr_item_qty#16] -(31) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] +(27) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#19)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#15)] PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 10] -Input [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] - -(33) Filter [codegen id : 10] -Input [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] -Condition : isnotnull(cr_item_sk#17) - -(34) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#20, i_item_id#21] - -(35) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cr_item_sk#17] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 10] -Output [3]: [cr_return_quantity#18, cr_returned_date_sk#19, i_item_id#21] -Input [5]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19, i_item_sk#20, i_item_id#21] - -(37) ReusedExchange [Reuses operator id: 25] -Output [1]: [d_date_sk#22] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cr_returned_date_sk#19] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [2]: [cr_return_quantity#18, i_item_id#21] -Input [4]: [cr_return_quantity#18, cr_returned_date_sk#19, i_item_id#21, d_date_sk#22] - -(40) HashAggregate [codegen id : 10] -Input [2]: [cr_return_quantity#18, i_item_id#21] -Keys [1]: [i_item_id#21] -Functions [1]: [partial_sum(cr_return_quantity#18)] -Aggregate Attributes [1]: [sum#23] -Results [2]: [i_item_id#21, sum#24] - -(41) Exchange -Input [2]: [i_item_id#21, sum#24] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [2]: [i_item_id#21, sum#24] -Keys [1]: [i_item_id#21] -Functions [1]: [sum(cr_return_quantity#18)] -Aggregate Attributes [1]: [sum(cr_return_quantity#18)#25] -Results [2]: [i_item_id#21 AS item_id#26, sum(cr_return_quantity#18)#25 AS cr_item_qty#27] - -(43) BroadcastExchange -Input [2]: [item_id#26, cr_item_qty#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#26] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 18] -Output [3]: [item_id#15, sr_item_qty#16, cr_item_qty#27] -Input [4]: [item_id#15, sr_item_qty#16, item_id#26, cr_item_qty#27] - -(46) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] +(28) CometFilter +Input [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] +Condition : isnotnull(cr_item_sk#13) + +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#16, i_item_id#17] + +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] +Right output [2]: [i_item_sk#16, i_item_id#17] +Arguments: [cr_item_sk#13], [i_item_sk#16], Inner, BuildRight + +(31) CometProject +Input [5]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15, i_item_sk#16, i_item_id#17] +Arguments: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17], [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] + +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_date#19] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#30)] +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [d_date_sk#18, d_date#19] +Condition : isnotnull(d_date_sk#18) + +(34) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(35) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_week_seq#9#20] + +(36) CometBroadcastHashJoin +Left output [2]: [d_date#8, d_week_seq#9] +Right output [1]: [d_week_seq#9#20] +Arguments: [d_week_seq#9], [d_week_seq#9#20], LeftSemi, BuildRight + +(37) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8], [d_date#8] + +(38) CometBroadcastExchange +Input [1]: [d_date#8] +Arguments: [d_date#8] + +(39) CometBroadcastHashJoin +Left output [2]: [d_date_sk#18, d_date#19] +Right output [1]: [d_date#8] +Arguments: [d_date#19], [d_date#8], LeftSemi, BuildRight + +(40) CometProject +Input [2]: [d_date_sk#18, d_date#19] +Arguments: [d_date_sk#18], [d_date_sk#18] + +(41) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] + +(42) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] +Right output [1]: [d_date_sk#18] +Arguments: [cr_returned_date_sk#15], [d_date_sk#18], Inner, BuildRight + +(43) CometProject +Input [4]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17, d_date_sk#18] +Arguments: [cr_return_quantity#14, i_item_id#17], [cr_return_quantity#14, i_item_id#17] + +(44) CometHashAggregate +Input [2]: [cr_return_quantity#14, i_item_id#17] +Keys [1]: [i_item_id#17] +Functions [1]: [partial_sum(cr_return_quantity#14)] + +(45) CometExchange +Input [2]: [i_item_id#17, sum#21] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(46) CometHashAggregate +Input [2]: [i_item_id#17, sum#21] +Keys [1]: [i_item_id#17] +Functions [1]: [sum(cr_return_quantity#14)] + +(47) CometBroadcastExchange +Input [2]: [item_id#22, cr_item_qty#23] +Arguments: [item_id#22, cr_item_qty#23] + +(48) CometBroadcastHashJoin +Left output [2]: [item_id#24, sr_item_qty#25] +Right output [2]: [item_id#22, cr_item_qty#23] +Arguments: [item_id#24], [item_id#22], Inner, BuildRight + +(49) CometProject +Input [4]: [item_id#24, sr_item_qty#25, item_id#22, cr_item_qty#23] +Arguments: [item_id#24, sr_item_qty#25, cr_item_qty#23], [item_id#24, sr_item_qty#25, cr_item_qty#23] + +(50) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#28)] PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 16] -Input [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] - -(48) Filter [codegen id : 16] -Input [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] -Condition : isnotnull(wr_item_sk#28) - -(49) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#31, i_item_id#32] - -(50) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [wr_item_sk#28] -Right keys [1]: [i_item_sk#31] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 16] -Output [3]: [wr_return_quantity#29, wr_returned_date_sk#30, i_item_id#32] -Input [5]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30, i_item_sk#31, i_item_id#32] - -(52) ReusedExchange [Reuses operator id: 25] -Output [1]: [d_date_sk#33] - -(53) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [wr_returned_date_sk#30] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(54) Project [codegen id : 16] -Output [2]: [wr_return_quantity#29, i_item_id#32] -Input [4]: [wr_return_quantity#29, wr_returned_date_sk#30, i_item_id#32, d_date_sk#33] - -(55) HashAggregate [codegen id : 16] -Input [2]: [wr_return_quantity#29, i_item_id#32] -Keys [1]: [i_item_id#32] -Functions [1]: [partial_sum(wr_return_quantity#29)] -Aggregate Attributes [1]: [sum#34] -Results [2]: [i_item_id#32, sum#35] - -(56) Exchange -Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(57) HashAggregate [codegen id : 17] -Input [2]: [i_item_id#32, sum#35] -Keys [1]: [i_item_id#32] -Functions [1]: [sum(wr_return_quantity#29)] -Aggregate Attributes [1]: [sum(wr_return_quantity#29)#36] -Results [2]: [i_item_id#32 AS item_id#37, sum(wr_return_quantity#29)#36 AS wr_item_qty#38] - -(58) BroadcastExchange -Input [2]: [item_id#37, wr_item_qty#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#37] -Join type: Inner -Join condition: None - -(60) Project [codegen id : 18] -Output [8]: [item_id#15, sr_item_qty#16, (((cast(sr_item_qty#16 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS sr_dev#39, cr_item_qty#27, (((cast(cr_item_qty#27 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS cr_dev#40, wr_item_qty#38, (((cast(wr_item_qty#38 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS wr_dev#41, (cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as decimal(20,0)) / 3.0) AS average#42] -Input [5]: [item_id#15, sr_item_qty#16, cr_item_qty#27, item_id#37, wr_item_qty#38] - -(61) TakeOrderedAndProject -Input [8]: [item_id#15, sr_item_qty#16, sr_dev#39, cr_item_qty#27, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] -Arguments: 100, [item_id#15 ASC NULLS FIRST, sr_item_qty#16 ASC NULLS FIRST], [item_id#15, sr_item_qty#16, sr_dev#39, cr_item_qty#27, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] +(51) CometFilter +Input [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] +Condition : isnotnull(wr_item_sk#26) + +(52) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#29, i_item_id#30] + +(53) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] +Right output [2]: [i_item_sk#29, i_item_id#30] +Arguments: [wr_item_sk#26], [i_item_sk#29], Inner, BuildRight + +(54) CometProject +Input [5]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28, i_item_sk#29, i_item_id#30] +Arguments: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30], [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30] + +(55) ReusedExchange [Reuses operator id: 41] +Output [1]: [d_date_sk#31] + +(56) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30] +Right output [1]: [d_date_sk#31] +Arguments: [wr_returned_date_sk#28], [d_date_sk#31], Inner, BuildRight + +(57) CometProject +Input [4]: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30, d_date_sk#31] +Arguments: [wr_return_quantity#27, i_item_id#30], [wr_return_quantity#27, i_item_id#30] + +(58) CometHashAggregate +Input [2]: [wr_return_quantity#27, i_item_id#30] +Keys [1]: [i_item_id#30] +Functions [1]: [partial_sum(wr_return_quantity#27)] + +(59) CometExchange +Input [2]: [i_item_id#30, sum#32] +Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(60) CometHashAggregate +Input [2]: [i_item_id#30, sum#32] +Keys [1]: [i_item_id#30] +Functions [1]: [sum(wr_return_quantity#27)] + +(61) CometBroadcastExchange +Input [2]: [item_id#33, wr_item_qty#34] +Arguments: [item_id#33, wr_item_qty#34] + +(62) CometBroadcastHashJoin +Left output [3]: [item_id#24, sr_item_qty#25, cr_item_qty#23] +Right output [2]: [item_id#33, wr_item_qty#34] +Arguments: [item_id#24], [item_id#33], Inner, BuildRight + +(63) CometProject +Input [5]: [item_id#24, sr_item_qty#25, cr_item_qty#23, item_id#33, wr_item_qty#34] +Arguments: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], [item_id#24, sr_item_qty#25, (((cast(sr_item_qty#25 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS sr_dev#35, cr_item_qty#23, (((cast(cr_item_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS cr_dev#36, wr_item_qty#34, (((cast(wr_item_qty#34 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS wr_dev#37, (cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as decimal(20,0)) / 3.0) AS average#38] + +(64) CometTakeOrderedAndProject +Input [8]: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#24 ASC NULLS FIRST,sr_item_qty#25 ASC NULLS FIRST], output=[item_id#24,sr_item_qty#25,sr_dev#35,cr_item_qty#23,cr_dev#36,wr_item_qty#34,wr_dev#37,average#38]), [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], 100, 0, [item_id#24 ASC NULLS FIRST, sr_item_qty#25 ASC NULLS FIRST], [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] + +(65) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/extended.txt new file mode 100644 index 0000000000..742a696f6f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/extended.txt @@ -0,0 +1,88 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 85 out of 85 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/simplified.txt index b186ed0d51..6008f36f19 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/simplified.txt @@ -1,91 +1,67 @@ -TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] - WholeStageCodegen (18) - Project [item_id,sr_item_qty,cr_item_qty,wr_item_qty] - BroadcastHashJoin [item_id,item_id] - Project [item_id,sr_item_qty,cr_item_qty] - BroadcastHashJoin [item_id,item_id] - HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,sr_return_quantity] [sum,sum] - Project [sr_return_quantity,i_item_id] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [sr_return_quantity,sr_returned_date_sk,i_item_id] - BroadcastHashJoin [sr_item_sk,i_item_sk] - Filter [sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [d_date_sk] - BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date] - BroadcastHashJoin [d_week_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (11) - HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (10) - HashAggregate [i_item_id,cr_return_quantity] [sum,sum] - Project [cr_return_quantity,i_item_id] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Project [cr_return_quantity,cr_returned_date_sk,i_item_id] - BroadcastHashJoin [cr_item_sk,i_item_sk] - Filter [cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,sum] [sum(wr_return_quantity),item_id,wr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #9 - WholeStageCodegen (16) - HashAggregate [i_item_id,wr_return_quantity] [sum,sum] - Project [wr_return_quantity,i_item_id] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Project [wr_return_quantity,wr_returned_date_sk,i_item_id] - BroadcastHashJoin [wr_item_sk,i_item_sk] - Filter [wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometBroadcastHashJoin [item_id,sr_item_qty,cr_item_qty,item_id,wr_item_qty] + CometProject [item_id,sr_item_qty,cr_item_qty] + CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] + CometHashAggregate [sum] [item_id,sr_item_qty,i_item_id,sum(sr_return_quantity)] + CometExchange [i_item_id] #1 + CometHashAggregate [sr_return_quantity] [i_item_id,sum] + CometProject [sr_return_quantity,i_item_id] + CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] + CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id] #2 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #4 + CometProject [d_date] [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #5 + CometProject [d_week_seq] [d_week_seq] + CometFilter [d_date,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cr_item_qty] #6 + CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] + CometExchange [i_item_id] #7 + CometHashAggregate [cr_return_quantity] [i_item_id,sum] + CometProject [cr_return_quantity,i_item_id] + CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] + CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #9 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + ReusedExchange [d_week_seq] #5 + CometBroadcastExchange [item_id,wr_item_qty] #10 + CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] + CometExchange [i_item_id] #11 + CometHashAggregate [wr_return_quantity] [i_item_id,sum] + CometProject [wr_return_quantity,i_item_id] + CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] + CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/explain.txt index 3ffa871d0b..63140cfc2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/explain.txt @@ -1,210 +1,190 @@ == Physical Plan == -TakeOrderedAndProject (37) -+- * Project (36) - +- * BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer_address (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_demographics (11) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.household_demographics (17) - : +- BroadcastExchange (27) - : +- * Project (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.income_band (23) - +- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) - +- Scan parquet spark_catalog.default.store_returns (31) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (10) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.income_band (20) + +- CometProject (29) + +- CometFilter (28) + +- CometNativeScan parquet spark_catalog.default.store_returns (27) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) -(3) Filter [codegen id : 5] +(3) CometProject Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#1, 16, true, false, true) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#5, 20, true, false, true) AS c_first_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#6, 30, true, false, true) AS c_last_name#9] -(4) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#7, ca_city#8] +(4) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#10, ca_city#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [ca_address_sk#7, ca_city#8] - -(6) Filter [codegen id : 1] -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) +(5) CometFilter +Input [2]: [ca_address_sk#10, ca_city#11] +Condition : ((isnotnull(ca_city#11) AND (ca_city#11 = Edgewood)) AND isnotnull(ca_address_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [ca_address_sk#7] -Input [2]: [ca_address_sk#7, ca_city#8] +(6) CometProject +Input [2]: [ca_address_sk#10, ca_city#11] +Arguments: [ca_address_sk#10], [ca_address_sk#10] -(8) BroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(7) CometBroadcastExchange +Input [1]: [ca_address_sk#10] +Arguments: [ca_address_sk#10] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#4] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9] +Right output [1]: [ca_address_sk#10] +Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight -(10) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] +(9) CometProject +Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(11) Scan parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#9] +(10) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [cd_demo_sk#9] +(11) CometFilter +Input [1]: [cd_demo_sk#12] +Condition : isnotnull(cd_demo_sk#12) -(13) Filter [codegen id : 2] -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) +(12) CometBroadcastExchange +Input [1]: [cd_demo_sk#12] +Arguments: [cd_demo_sk#12] -(14) BroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] +Right output [1]: [cd_demo_sk#12] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(14) CometProject +Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(16) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(17) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] +(15) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] - -(19) Filter [codegen id : 3] -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) +(16) CometFilter +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Condition : (isnotnull(hd_demo_sk#13) AND isnotnull(hd_income_band_sk#14)) -(20) BroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [hd_demo_sk#13, hd_income_band_sk#14] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#10] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight -(22) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] +(19) CometProject +Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(23) Scan parquet spark_catalog.default.income_band -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] +(20) CometNativeScan parquet spark_catalog.default.income_band +Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(25) Filter [codegen id : 4] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) +(21) CometFilter +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Condition : ((((isnotnull(ib_lower_bound#16) AND isnotnull(ib_upper_bound#17)) AND (ib_lower_bound#16 >= 38128)) AND (ib_upper_bound#17 <= 88128)) AND isnotnull(ib_income_band_sk#15)) -(26) Project [codegen id : 4] -Output [1]: [ib_income_band_sk#12] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] +(22) CometProject +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Arguments: [ib_income_band_sk#15], [ib_income_band_sk#15] -(27) BroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [1]: [ib_income_band_sk#15] +Arguments: [ib_income_band_sk#15] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [hd_income_band_sk#11] -Right keys [1]: [ib_income_band_sk#12] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] +Right output [1]: [ib_income_band_sk#15] +Arguments: [hd_income_band_sk#14], [ib_income_band_sk#15], Inner, BuildRight -(29) Project [codegen id : 5] -Output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] +(25) CometProject +Input [6]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14, ib_income_band_sk#15] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(30) BroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[3, int, true] as bigint)),false), [plan_id=5] +(26) CometBroadcastExchange +Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(31) Scan parquet spark_catalog.default.store_returns -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +(27) CometNativeScan parquet spark_catalog.default.store_returns +Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_cdemo_sk)] ReadSchema: struct -(32) ColumnarToRow -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +(28) CometFilter +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_cdemo_sk#18) -(33) Filter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) +(29) CometProject +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Arguments: [sr_cdemo_sk#18], [sr_cdemo_sk#18] -(34) Project -Output [1]: [sr_cdemo_sk#15] -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +(30) CometBroadcastHashJoin +Left output [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [1]: [sr_cdemo_sk#18] +Arguments: [cd_demo_sk#12], [sr_cdemo_sk#18], Inner, BuildLeft -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cd_demo_sk#9] -Right keys [1]: [sr_cdemo_sk#15] -Join type: Inner -Join condition: None +(31) CometProject +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] +Arguments: [customer_id#20, customername#21, c_customer_id#7], [c_customer_id#7 AS customer_id#20, concat(c_last_name#9, , , c_first_name#8) AS customername#21, c_customer_id#7] -(36) Project [codegen id : 6] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] +(32) CometTakeOrderedAndProject +Input [3]: [customer_id#20, customername#21, c_customer_id#7] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#7 ASC NULLS FIRST], output=[customer_id#20,customername#21]), [customer_id#20, customername#21], 100, 0, [c_customer_id#7 ASC NULLS FIRST], [customer_id#20, customername#21] -(37) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] +(33) CometColumnarToRow [codegen id : 1] +Input [2]: [customer_id#20, customername#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/extended.txt new file mode 100644 index 0000000000..b025c868d1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store_returns + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/simplified.txt index c42e500847..2f4baa9f3f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/simplified.txt @@ -1,54 +1,35 @@ -TakeOrderedAndProject [c_customer_id,customer_id,customername] - WholeStageCodegen (6) - Project [c_customer_id,c_last_name,c_first_name] - BroadcastHashJoin [cd_demo_sk,sr_cdemo_sk] - InputAdapter - BroadcastExchange #1 - WholeStageCodegen (5) - Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [ca_address_sk] - Filter [ca_city,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [ib_income_band_sk] - Filter [ib_lower_bound,ib_upper_bound,ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - Project [sr_cdemo_sk] - Filter [sr_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customername,c_customer_id] + CometProject [c_last_name,c_first_name] [customer_id,customername,c_customer_id] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] + CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 + CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk,ib_income_band_sk] + CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] + CometBroadcastHashJoin [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk,hd_demo_sk,hd_income_band_sk] + CometProject [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] + CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] + CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometBroadcastExchange [ca_address_sk] #2 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_city] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometBroadcastExchange [cd_demo_sk] #3 + CometFilter [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 + CometFilter [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometBroadcastExchange [ib_income_band_sk] #5 + CometProject [ib_income_band_sk] + CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometProject [sr_cdemo_sk] + CometFilter [sr_cdemo_sk,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/explain.txt index b534d36146..d4ba3e2a08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/explain.txt @@ -1,300 +1,276 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin Inner BuildRight (47) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (28) - : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (16) - : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : :- BroadcastExchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : : : +- * Project (8) - : : : : : : +- * Filter (7) - : : : : : : +- * ColumnarToRow (6) - : : : : : : +- Scan parquet spark_catalog.default.web_returns (5) - : : : : : +- BroadcastExchange (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.web_page (11) - : : : : +- BroadcastExchange (20) - : : : : +- * Filter (19) - : : : : +- * ColumnarToRow (18) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (17) - : : : +- BroadcastExchange (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.customer_demographics (23) - : : +- BroadcastExchange (33) - : : +- * Project (32) - : : +- * Filter (31) - : : +- * ColumnarToRow (30) - : : +- Scan parquet spark_catalog.default.customer_address (29) - : +- BroadcastExchange (40) - : +- * Project (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) - : +- Scan parquet spark_catalog.default.date_dim (36) - +- BroadcastExchange (46) - +- * Filter (45) - +- * ColumnarToRow (44) - +- Scan parquet spark_catalog.default.reason (43) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (48) ++- CometTakeOrderedAndProject (47) + +- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometBroadcastHashJoin (42) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometBroadcastExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : : : +- CometProject (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometProject (16) + : : : : +- CometFilter (15) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (14) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.date_dim (32) + +- CometBroadcastExchange (41) + +- CometProject (40) + +- CometFilter (39) + +- CometNativeScan parquet spark_catalog.default.reason (38) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00)))) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[2, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(5) Scan parquet spark_catalog.default.web_returns +(4) CometNativeScan parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] ReadSchema: struct -(6) ColumnarToRow -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] - -(7) Filter +(5) CometFilter Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) -(8) Project -Output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +(6) CometProject Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [ws_item_sk#1, ws_order_number#3] -Right keys [2]: [wr_item_sk#8, wr_order_number#13] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft -(10) Project [codegen id : 8] -Output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +(8) CometProject Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(11) Scan parquet spark_catalog.default.web_page +(9) CometNativeScan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [wp_web_page_sk#17] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [1]: [wp_web_page_sk#17] Condition : isnotnull(wp_web_page_sk#17) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [wp_web_page_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [wp_web_page_sk#17] -(15) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_web_page_sk#2] -Right keys [1]: [wp_web_page_sk#17] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [wp_web_page_sk#17] +Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight -(16) Project [codegen id : 8] -Output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +(13) CometProject Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] +Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(17) Scan parquet spark_catalog.default.customer_demographics +(14) CometNativeScan parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(19) Filter [codegen id : 3] +(16) CometProject Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(20) BroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_refunded_cdemo_sk#9] -Right keys [1]: [cd_demo_sk#18] -Join type: Inner -Join condition: ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))) +(18) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight -(22) Project [codegen id : 8] -Output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(19) CometProject +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] -(23) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(20) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(21) CometFilter +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true))) -(25) Filter [codegen id : 4] -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) +(22) CometProject +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#27] -(26) BroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], input[1, string, false], input[2, string, false]),false), [plan_id=4] +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] -(27) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20] -Right keys [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight -(28) Project [codegen id : 8] -Output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(25) CometProject +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(29) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] +(26) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,NJ,OH]),In(ca_state, [CT,KY,WI])),In(ca_state, [AR,IA,LA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] - -(31) Filter [codegen id : 5] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) +(27) CometFilter +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (LA,IA,AR))) -(32) Project [codegen id : 5] -Output [2]: [ca_address_sk#24, ca_state#25] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] +(28) CometProject +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) AS ca_state#31] -(33) BroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_refunded_addr_sk#10] -Right keys [1]: [ca_address_sk#24] -Join type: Inner -Join condition: ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))) +(30) CometBroadcastHashJoin +Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight -(35) Project [codegen id : 8] -Output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] +(31) CometProject +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] +Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(36) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#27, d_year#28] +(32) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [2]: [d_date_sk#27, d_year#28] +(33) CometFilter +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) -(38) Filter [codegen id : 6] -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +(34) CometProject +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] -(39) Project [codegen id : 6] -Output [1]: [d_date_sk#27] -Input [2]: [d_date_sk#27, d_year#28] +(35) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(40) BroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(36) CometBroadcastHashJoin +Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_sold_date_sk#7] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None +(37) CometProject +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] +Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(42) Project [codegen id : 8] -Output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] - -(43) Scan parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#29, r_reason_desc#30] +(38) CometNativeScan parquet spark_catalog.default.reason +Output [2]: [r_reason_sk#34, r_reason_desc#35] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [2]: [r_reason_sk#29, r_reason_desc#30] +(39) CometFilter +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) -(45) Filter [codegen id : 7] -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) +(40) CometProject +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#35, 100, true, false, true) AS r_reason_desc#36] -(46) BroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(41) CometBroadcastExchange +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] -(47) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_reason_sk#12] -Right keys [1]: [r_reason_sk#29] -Join type: Inner -Join condition: None +(42) CometBroadcastHashJoin +Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight -(48) Project [codegen id : 8] -Output [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] +(43) CometProject +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] -(49) HashAggregate [codegen id : 8] -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] +(44) CometHashAggregate +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [6]: [sum#31, count#32, sum#33, count#34, sum#35, count#36] -Results [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -(50) Exchange -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(45) CometExchange +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(51) HashAggregate [codegen id : 9] -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Keys [1]: [r_reason_desc#30] +(46) CometHashAggregate +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [3]: [avg(ws_quantity#4)#43, avg(UnscaledValue(wr_refunded_cash#15))#44, avg(UnscaledValue(wr_fee#14))#45] -Results [4]: [substr(r_reason_desc#30, 1, 20) AS substr(r_reason_desc, 1, 20)#46, avg(ws_quantity#4)#43 AS avg(ws_quantity)#47, cast((avg(UnscaledValue(wr_refunded_cash#15))#44 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#48, cast((avg(UnscaledValue(wr_fee#14))#45 / 100.0) as decimal(11,6)) AS avg(wr_fee)#49] -(52) TakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] -Arguments: 100, [substr(r_reason_desc, 1, 20)#46 ASC NULLS FIRST, avg(ws_quantity)#47 ASC NULLS FIRST, avg(wr_refunded_cash)#48 ASC NULLS FIRST, avg(wr_fee)#49 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] +(47) CometTakeOrderedAndProject +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] + +(48) CometColumnarToRow [codegen id : 1] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/extended.txt new file mode 100644 index 0000000000..358da39c56 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/extended.txt @@ -0,0 +1,50 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_page + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 47 out of 47 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/simplified.txt index c5a45b42cb..925b2ed9b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/simplified.txt @@ -1,77 +1,50 @@ -TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - WholeStageCodegen (9) - HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count] - InputAdapter - Exchange [r_reason_desc] #1 - WholeStageCodegen (8) - HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] - BroadcastHashJoin [wr_reason_sk,r_reason_sk] - Project [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [wr_refunded_addr_sk,ca_address_sk,ca_state,ws_net_profit] - Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [wr_returning_cdemo_sk,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] - Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] - BroadcastHashJoin [wr_refunded_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ws_sales_price] - Project [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ws_item_sk,ws_order_number,ws_web_page_sk,ws_sales_price,ws_net_profit] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - Filter [wr_item_sk,wr_order_number,wr_refunded_cdemo_sk,wr_returning_cdemo_sk,wr_refunded_addr_sk,wr_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] + CometHashAggregate [sum,count,sum,count,sum,count] [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] + CometExchange [r_reason_desc] #1 + CometHashAggregate [ws_quantity,wr_refunded_cash,wr_fee] [r_reason_desc,sum,count,sum,count,sum,count] + CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] + CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] + CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] + CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 + CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometBroadcastExchange [wp_web_page_sk] #3 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #6 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 + CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] + CometFilter [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/explain.txt index caf90212ac..e0b9dfa394 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/explain.txt @@ -1,145 +1,136 @@ == Physical Plan == -TakeOrderedAndProject (25) -+- * Project (24) - +- Window (23) - +- * Sort (22) - +- Exchange (21) - +- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Expand (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- BroadcastExchange (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometExchange (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometExpand (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.item (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ws_item_sk#1, ws_net_paid#2] +(8) CometProject Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] +Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] -(16) Project [codegen id : 3] -Output [3]: [ws_net_paid#2, i_category#8, i_class#7] -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] +(13) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#1, ws_net_paid#2] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(17) Expand [codegen id : 3] -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] -(18) HashAggregate [codegen id : 3] -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(15) CometExpand +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] + +(16) CometHashAggregate +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#12] -Results [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -(19) Exchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(18) CometHashAggregate +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#14] -Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS total_sum#15, i_category#9, i_class#10, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS lochierarchy#16, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS _w0#17, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS _w1#18, CASE WHEN (cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint) = 0) THEN i_category#9 END AS _w2#19] -(21) Exchange -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19], [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(22) Sort [codegen id : 5] -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST], false, 0 +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] -(23) Window -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] +(22) Window +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] -(24) Project [codegen id : 6] -Output [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Input [8]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] +(23) Project [codegen id : 2] +Output [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Input [8]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] -(25) TakeOrderedAndProject -Input [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] +(24) TakeOrderedAndProject +Input [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/extended.txt new file mode 100644 index 0000000000..cf1ba69679 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/extended.txt @@ -0,0 +1,26 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometExpand + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 20 out of 23 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/simplified.txt index 8ae1bf14f1..81a2237a29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/simplified.txt @@ -1,39 +1,28 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (5) - Sort [_w1,_w2,_w0] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (3) - HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum] - Expand [ws_net_paid,i_category,i_class] - Project [ws_net_paid,i_category,i_class] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_net_paid] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometExchange [_w1,_w2] #1 + CometHashAggregate [sum] [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ws_net_paid)),spark_grouping_id] + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ws_net_paid] [i_category,i_class,spark_grouping_id,sum] + CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] + CometProject [ws_net_paid,i_category,i_class] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/explain.txt index 3fb94a8661..c721c81f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/explain.txt @@ -1,307 +1,290 @@ == Physical Plan == * HashAggregate (51) -+- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin LeftAnti BuildRight (47) - :- * BroadcastHashJoin LeftAnti BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - +- BroadcastExchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales ++- * CometColumnarToRow (50) + +- CometColumnarExchange (49) + +- * HashAggregate (48) + +- * Project (47) + +- * BroadcastHashJoin LeftAnti BuildRight (46) + :- * BroadcastHashJoin LeftAnti BuildRight (32) + : :- * CometColumnarToRow (18) + : : +- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (31) + : +- * CometColumnarToRow (30) + : +- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometFilter (20) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (19) + : : +- ReusedExchange (21) + : +- ReusedExchange (24) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : +- ReusedExchange (35) + +- ReusedExchange (38) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#3, d_date#4] +(5) CometProject Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#3, d_date#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#3, d_date#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#2] -Right keys [1]: [d_date_sk#3] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ss_customer_sk#1, d_date#4] +(8) CometProject Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(11) Scan parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(17) HashAggregate [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(18) Exchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 12] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(18) CometColumnarToRow [codegen id : 3] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] + +(19) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(20) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) -(22) Filter [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#11, d_date#12] +(22) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(23) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(25) Project [codegen id : 6] -Output [2]: [cs_bill_customer_sk#9, d_date#12] -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] +(24) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] -(26) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] +(25) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_bill_customer_sk#9] -Right keys [1]: [c_customer_sk#13] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 6] -Output [3]: [c_last_name#15, c_first_name#14, d_date#12] -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] +(26) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] -(29) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(27) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(30) Exchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) HashAggregate [codegen id : 7] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(29) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(32) BroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -(33) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)] +(31) BroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] + +(32) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None -(34) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] - -(36) Filter [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) +(34) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(37) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#18, d_date#19] +(35) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#17] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(39) Project [codegen id : 10] -Output [2]: [ws_bill_customer_sk#16, d_date#19] -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] +(37) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] -(40) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(38) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#16] -Right keys [1]: [c_customer_sk#20] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight -(42) Project [codegen id : 10] -Output [3]: [c_last_name#22, c_first_name#21, d_date#19] -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] +(40) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] -(43) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(44) Exchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(42) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(45) HashAggregate [codegen id : 11] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(43) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(46) BroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=7] +(44) CometColumnarToRow [codegen id : 2] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] -(47) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)] +(45) BroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] + +(46) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None -(48) Project [codegen id : 12] +(47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(49) HashAggregate [codegen id : 12] +(48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] + +(49) CometColumnarExchange +Input [1]: [count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(50) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [count#26] -(51) HashAggregate [codegen id : 13] -Input [1]: [count#24] +(51) HashAggregate [codegen id : 4] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS count(1)#26] +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/extended.txt new file mode 100644 index 0000000000..f659e21953 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/extended.txt @@ -0,0 +1,65 @@ +HashAggregate ++- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 52 out of 59 eligible operators (88%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/simplified.txt index 9122ac943b..0fe68e3efa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/simplified.txt @@ -1,77 +1,61 @@ -WholeStageCodegen (13) +WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (12) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - HashAggregate [c_last_name,c_first_name,d_date] + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk,d_date] #3 - InputAdapter ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedExchange [d_date_sk,d_date] #3 - InputAdapter ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/explain.txt index be540f124f..25b4c305f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/explain.txt @@ -1,1031 +1,927 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (182) -:- * BroadcastNestedLoopJoin Inner BuildRight (160) -: :- * BroadcastNestedLoopJoin Inner BuildRight (138) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (116) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (94) -: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (72) -: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (50) -: : : : : : :- * HashAggregate (28) -: : : : : : : +- Exchange (27) -: : : : : : : +- * HashAggregate (26) -: : : : : : : +- * Project (25) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (24) -: : : : : : : :- * Project (18) -: : : : : : : : +- * BroadcastHashJoin Inner BuildRight (17) -: : : : : : : : :- * Project (11) -: : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (10) -: : : : : : : : : :- * Project (4) -: : : : : : : : : : +- * Filter (3) -: : : : : : : : : : +- * ColumnarToRow (2) -: : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) -: : : : : : : : : +- BroadcastExchange (9) -: : : : : : : : : +- * Project (8) -: : : : : : : : : +- * Filter (7) -: : : : : : : : : +- * ColumnarToRow (6) -: : : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (5) -: : : : : : : : +- BroadcastExchange (16) -: : : : : : : : +- * Project (15) -: : : : : : : : +- * Filter (14) -: : : : : : : : +- * ColumnarToRow (13) -: : : : : : : : +- Scan parquet spark_catalog.default.time_dim (12) -: : : : : : : +- BroadcastExchange (23) -: : : : : : : +- * Project (22) -: : : : : : : +- * Filter (21) -: : : : : : : +- * ColumnarToRow (20) -: : : : : : : +- Scan parquet spark_catalog.default.store (19) -: : : : : : +- BroadcastExchange (49) -: : : : : : +- * HashAggregate (48) -: : : : : : +- Exchange (47) -: : : : : : +- * HashAggregate (46) -: : : : : : +- * Project (45) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (44) -: : : : : : :- * Project (42) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) -: : : : : : : :- * Project (35) -: : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) -: : : : : : : : :- * Project (32) -: : : : : : : : : +- * Filter (31) -: : : : : : : : : +- * ColumnarToRow (30) -: : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (29) -: : : : : : : : +- ReusedExchange (33) -: : : : : : : +- BroadcastExchange (40) -: : : : : : : +- * Project (39) -: : : : : : : +- * Filter (38) -: : : : : : : +- * ColumnarToRow (37) -: : : : : : : +- Scan parquet spark_catalog.default.time_dim (36) -: : : : : : +- ReusedExchange (43) -: : : : : +- BroadcastExchange (71) -: : : : : +- * HashAggregate (70) -: : : : : +- Exchange (69) -: : : : : +- * HashAggregate (68) -: : : : : +- * Project (67) -: : : : : +- * BroadcastHashJoin Inner BuildRight (66) -: : : : : :- * Project (64) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (63) -: : : : : : :- * Project (57) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) -: : : : : : : :- * Project (54) -: : : : : : : : +- * Filter (53) -: : : : : : : : +- * ColumnarToRow (52) -: : : : : : : : +- Scan parquet spark_catalog.default.store_sales (51) -: : : : : : : +- ReusedExchange (55) -: : : : : : +- BroadcastExchange (62) -: : : : : : +- * Project (61) -: : : : : : +- * Filter (60) -: : : : : : +- * ColumnarToRow (59) -: : : : : : +- Scan parquet spark_catalog.default.time_dim (58) -: : : : : +- ReusedExchange (65) -: : : : +- BroadcastExchange (93) -: : : : +- * HashAggregate (92) -: : : : +- Exchange (91) -: : : : +- * HashAggregate (90) -: : : : +- * Project (89) -: : : : +- * BroadcastHashJoin Inner BuildRight (88) -: : : : :- * Project (86) -: : : : : +- * BroadcastHashJoin Inner BuildRight (85) -: : : : : :- * Project (79) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (78) -: : : : : : :- * Project (76) -: : : : : : : +- * Filter (75) -: : : : : : : +- * ColumnarToRow (74) -: : : : : : : +- Scan parquet spark_catalog.default.store_sales (73) -: : : : : : +- ReusedExchange (77) -: : : : : +- BroadcastExchange (84) -: : : : : +- * Project (83) -: : : : : +- * Filter (82) -: : : : : +- * ColumnarToRow (81) -: : : : : +- Scan parquet spark_catalog.default.time_dim (80) -: : : : +- ReusedExchange (87) -: : : +- BroadcastExchange (115) -: : : +- * HashAggregate (114) -: : : +- Exchange (113) -: : : +- * HashAggregate (112) -: : : +- * Project (111) -: : : +- * BroadcastHashJoin Inner BuildRight (110) -: : : :- * Project (108) -: : : : +- * BroadcastHashJoin Inner BuildRight (107) -: : : : :- * Project (101) -: : : : : +- * BroadcastHashJoin Inner BuildRight (100) -: : : : : :- * Project (98) -: : : : : : +- * Filter (97) -: : : : : : +- * ColumnarToRow (96) -: : : : : : +- Scan parquet spark_catalog.default.store_sales (95) -: : : : : +- ReusedExchange (99) -: : : : +- BroadcastExchange (106) -: : : : +- * Project (105) -: : : : +- * Filter (104) -: : : : +- * ColumnarToRow (103) -: : : : +- Scan parquet spark_catalog.default.time_dim (102) -: : : +- ReusedExchange (109) -: : +- BroadcastExchange (137) -: : +- * HashAggregate (136) -: : +- Exchange (135) -: : +- * HashAggregate (134) -: : +- * Project (133) -: : +- * BroadcastHashJoin Inner BuildRight (132) -: : :- * Project (130) -: : : +- * BroadcastHashJoin Inner BuildRight (129) -: : : :- * Project (123) -: : : : +- * BroadcastHashJoin Inner BuildRight (122) -: : : : :- * Project (120) -: : : : : +- * Filter (119) -: : : : : +- * ColumnarToRow (118) -: : : : : +- Scan parquet spark_catalog.default.store_sales (117) -: : : : +- ReusedExchange (121) -: : : +- BroadcastExchange (128) -: : : +- * Project (127) -: : : +- * Filter (126) -: : : +- * ColumnarToRow (125) -: : : +- Scan parquet spark_catalog.default.time_dim (124) -: : +- ReusedExchange (131) -: +- BroadcastExchange (159) -: +- * HashAggregate (158) -: +- Exchange (157) -: +- * HashAggregate (156) -: +- * Project (155) -: +- * BroadcastHashJoin Inner BuildRight (154) -: :- * Project (152) -: : +- * BroadcastHashJoin Inner BuildRight (151) -: : :- * Project (145) -: : : +- * BroadcastHashJoin Inner BuildRight (144) -: : : :- * Project (142) -: : : : +- * Filter (141) -: : : : +- * ColumnarToRow (140) -: : : : +- Scan parquet spark_catalog.default.store_sales (139) -: : : +- ReusedExchange (143) -: : +- BroadcastExchange (150) -: : +- * Project (149) -: : +- * Filter (148) -: : +- * ColumnarToRow (147) -: : +- Scan parquet spark_catalog.default.time_dim (146) -: +- ReusedExchange (153) -+- BroadcastExchange (181) - +- * HashAggregate (180) - +- Exchange (179) - +- * HashAggregate (178) - +- * Project (177) - +- * BroadcastHashJoin Inner BuildRight (176) - :- * Project (174) - : +- * BroadcastHashJoin Inner BuildRight (173) - : :- * Project (167) - : : +- * BroadcastHashJoin Inner BuildRight (166) - : : :- * Project (164) - : : : +- * Filter (163) - : : : +- * ColumnarToRow (162) - : : : +- Scan parquet spark_catalog.default.store_sales (161) - : : +- ReusedExchange (165) - : +- BroadcastExchange (172) - : +- * Project (171) - : +- * Filter (170) - : +- * ColumnarToRow (169) - : +- Scan parquet spark_catalog.default.time_dim (168) - +- ReusedExchange (175) - - -(1) Scan parquet spark_catalog.default.store_sales +* BroadcastNestedLoopJoin Inner BuildRight (172) +:- * BroadcastNestedLoopJoin Inner BuildRight (151) +: :- * BroadcastNestedLoopJoin Inner BuildRight (130) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (109) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) +: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) +: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) +: : : : : : :- * CometColumnarToRow (25) +: : : : : : : +- CometHashAggregate (24) +: : : : : : : +- CometExchange (23) +: : : : : : : +- CometHashAggregate (22) +: : : : : : : +- CometProject (21) +: : : : : : : +- CometBroadcastHashJoin (20) +: : : : : : : :- CometProject (15) +: : : : : : : : +- CometBroadcastHashJoin (14) +: : : : : : : : :- CometProject (9) +: : : : : : : : : +- CometBroadcastHashJoin (8) +: : : : : : : : : :- CometProject (3) +: : : : : : : : : : +- CometFilter (2) +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) +: : : : : : : : : +- CometBroadcastExchange (7) +: : : : : : : : : +- CometProject (6) +: : : : : : : : : +- CometFilter (5) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) +: : : : : : : : +- CometBroadcastExchange (13) +: : : : : : : : +- CometProject (12) +: : : : : : : : +- CometFilter (11) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) +: : : : : : : +- CometBroadcastExchange (19) +: : : : : : : +- CometProject (18) +: : : : : : : +- CometFilter (17) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store (16) +: : : : : : +- BroadcastExchange (45) +: : : : : : +- * CometColumnarToRow (44) +: : : : : : +- CometHashAggregate (43) +: : : : : : +- CometExchange (42) +: : : : : : +- CometHashAggregate (41) +: : : : : : +- CometProject (40) +: : : : : : +- CometBroadcastHashJoin (39) +: : : : : : :- CometProject (37) +: : : : : : : +- CometBroadcastHashJoin (36) +: : : : : : : :- CometProject (31) +: : : : : : : : +- CometBroadcastHashJoin (30) +: : : : : : : : :- CometProject (28) +: : : : : : : : : +- CometFilter (27) +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (26) +: : : : : : : : +- ReusedExchange (29) +: : : : : : : +- CometBroadcastExchange (35) +: : : : : : : +- CometProject (34) +: : : : : : : +- CometFilter (33) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (32) +: : : : : : +- ReusedExchange (38) +: : : : : +- BroadcastExchange (66) +: : : : : +- * CometColumnarToRow (65) +: : : : : +- CometHashAggregate (64) +: : : : : +- CometExchange (63) +: : : : : +- CometHashAggregate (62) +: : : : : +- CometProject (61) +: : : : : +- CometBroadcastHashJoin (60) +: : : : : :- CometProject (58) +: : : : : : +- CometBroadcastHashJoin (57) +: : : : : : :- CometProject (52) +: : : : : : : +- CometBroadcastHashJoin (51) +: : : : : : : :- CometProject (49) +: : : : : : : : +- CometFilter (48) +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (47) +: : : : : : : +- ReusedExchange (50) +: : : : : : +- CometBroadcastExchange (56) +: : : : : : +- CometProject (55) +: : : : : : +- CometFilter (54) +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (53) +: : : : : +- ReusedExchange (59) +: : : : +- BroadcastExchange (87) +: : : : +- * CometColumnarToRow (86) +: : : : +- CometHashAggregate (85) +: : : : +- CometExchange (84) +: : : : +- CometHashAggregate (83) +: : : : +- CometProject (82) +: : : : +- CometBroadcastHashJoin (81) +: : : : :- CometProject (79) +: : : : : +- CometBroadcastHashJoin (78) +: : : : : :- CometProject (73) +: : : : : : +- CometBroadcastHashJoin (72) +: : : : : : :- CometProject (70) +: : : : : : : +- CometFilter (69) +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (68) +: : : : : : +- ReusedExchange (71) +: : : : : +- CometBroadcastExchange (77) +: : : : : +- CometProject (76) +: : : : : +- CometFilter (75) +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim (74) +: : : : +- ReusedExchange (80) +: : : +- BroadcastExchange (108) +: : : +- * CometColumnarToRow (107) +: : : +- CometHashAggregate (106) +: : : +- CometExchange (105) +: : : +- CometHashAggregate (104) +: : : +- CometProject (103) +: : : +- CometBroadcastHashJoin (102) +: : : :- CometProject (100) +: : : : +- CometBroadcastHashJoin (99) +: : : : :- CometProject (94) +: : : : : +- CometBroadcastHashJoin (93) +: : : : : :- CometProject (91) +: : : : : : +- CometFilter (90) +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (89) +: : : : : +- ReusedExchange (92) +: : : : +- CometBroadcastExchange (98) +: : : : +- CometProject (97) +: : : : +- CometFilter (96) +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim (95) +: : : +- ReusedExchange (101) +: : +- BroadcastExchange (129) +: : +- * CometColumnarToRow (128) +: : +- CometHashAggregate (127) +: : +- CometExchange (126) +: : +- CometHashAggregate (125) +: : +- CometProject (124) +: : +- CometBroadcastHashJoin (123) +: : :- CometProject (121) +: : : +- CometBroadcastHashJoin (120) +: : : :- CometProject (115) +: : : : +- CometBroadcastHashJoin (114) +: : : : :- CometProject (112) +: : : : : +- CometFilter (111) +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (110) +: : : : +- ReusedExchange (113) +: : : +- CometBroadcastExchange (119) +: : : +- CometProject (118) +: : : +- CometFilter (117) +: : : +- CometNativeScan parquet spark_catalog.default.time_dim (116) +: : +- ReusedExchange (122) +: +- BroadcastExchange (150) +: +- * CometColumnarToRow (149) +: +- CometHashAggregate (148) +: +- CometExchange (147) +: +- CometHashAggregate (146) +: +- CometProject (145) +: +- CometBroadcastHashJoin (144) +: :- CometProject (142) +: : +- CometBroadcastHashJoin (141) +: : :- CometProject (136) +: : : +- CometBroadcastHashJoin (135) +: : : :- CometProject (133) +: : : : +- CometFilter (132) +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales (131) +: : : +- ReusedExchange (134) +: : +- CometBroadcastExchange (140) +: : +- CometProject (139) +: : +- CometFilter (138) +: : +- CometNativeScan parquet spark_catalog.default.time_dim (137) +: +- ReusedExchange (143) ++- BroadcastExchange (171) + +- * CometColumnarToRow (170) + +- CometHashAggregate (169) + +- CometExchange (168) + +- CometHashAggregate (167) + +- CometProject (166) + +- CometBroadcastHashJoin (165) + :- CometProject (163) + : +- CometBroadcastHashJoin (162) + : :- CometProject (157) + : : +- CometBroadcastHashJoin (156) + : : :- CometProject (154) + : : : +- CometFilter (153) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (152) + : : +- ReusedExchange (155) + : +- CometBroadcastExchange (161) + : +- CometProject (160) + : +- CometFilter (159) + : +- CometNativeScan parquet spark_catalog.default.time_dim (158) + +- ReusedExchange (164) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +(3) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(5) Scan parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(6) CometProject Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] -(9) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [hd_demo_sk#5] -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight -(11) Project [codegen id : 4] -Output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +(9) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] +Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(12) Scan parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 8)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#8] +(12) CometProject Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] +Arguments: [t_time_sk#8], [t_time_sk#8] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [t_time_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [t_time_sk#8] -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_time_sk#1] -Right keys [1]: [t_time_sk#8] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +Right output [1]: [t_time_sk#8] +Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight -(18) Project [codegen id : 4] -Output [1]: [ss_store_sk#3] +(15) CometProject Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] +Arguments: [ss_store_sk#3], [ss_store_sk#3] -(19) Scan parquet spark_catalog.default.store +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_store_name#12] - -(21) Filter [codegen id : 3] +(17) CometFilter Input [2]: [s_store_sk#11, s_store_name#12] Condition : ((isnotnull(s_store_name#12) AND (s_store_name#12 = ese)) AND isnotnull(s_store_sk#11)) -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#11] +(18) CometProject Input [2]: [s_store_sk#11, s_store_name#12] +Arguments: [s_store_sk#11], [s_store_sk#11] -(23) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#11] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#3] +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight -(25) Project [codegen id : 4] -Output: [] +(21) CometProject Input [2]: [ss_store_sk#3, s_store_sk#11] -(26) HashAggregate [codegen id : 4] +(22) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#13] -Results [1]: [count#14] -(27) Exchange -Input [1]: [count#14] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [1]: [count#13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 40] -Input [1]: [count#14] +(24) CometHashAggregate +Input [1]: [count#13] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#15] -Results [1]: [count(1)#15 AS h8_30_to_9#16] -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] +(25) CometColumnarToRow [codegen id : 8] +Input [1]: [h8_30_to_9#14] + +(26) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] +(27) CometFilter +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Condition : ((isnotnull(ss_hdemo_sk#16) AND isnotnull(ss_sold_time_sk#15)) AND isnotnull(ss_store_sk#17)) -(31) Filter [codegen id : 8] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] -Condition : ((isnotnull(ss_hdemo_sk#18) AND isnotnull(ss_sold_time_sk#17)) AND isnotnull(ss_store_sk#19)) +(28) CometProject +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17], [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] -(32) Project [codegen id : 8] -Output [3]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] +(29) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#19] -(33) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#21] +(30) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] +Right output [1]: [hd_demo_sk#19] +Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_hdemo_sk#18] -Right keys [1]: [hd_demo_sk#21] -Join type: Inner -Join condition: None +(31) CometProject +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] +Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(35) Project [codegen id : 8] -Output [2]: [ss_sold_time_sk#17, ss_store_sk#19] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, hd_demo_sk#21] - -(36) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#22, t_hour#23, t_minute#24] +(32) CometNativeScan parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] +(33) CometFilter +Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] +Condition : ((((isnotnull(t_hour#21) AND isnotnull(t_minute#22)) AND (t_hour#21 = 9)) AND (t_minute#22 < 30)) AND isnotnull(t_time_sk#20)) -(38) Filter [codegen id : 6] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] -Condition : ((((isnotnull(t_hour#23) AND isnotnull(t_minute#24)) AND (t_hour#23 = 9)) AND (t_minute#24 < 30)) AND isnotnull(t_time_sk#22)) +(34) CometProject +Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] +Arguments: [t_time_sk#20], [t_time_sk#20] -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#22] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] +(35) CometBroadcastExchange +Input [1]: [t_time_sk#20] +Arguments: [t_time_sk#20] -(40) BroadcastExchange -Input [1]: [t_time_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(36) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#15, ss_store_sk#17] +Right output [1]: [t_time_sk#20] +Arguments: [ss_sold_time_sk#15], [t_time_sk#20], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_time_sk#17] -Right keys [1]: [t_time_sk#22] -Join type: Inner -Join condition: None +(37) CometProject +Input [3]: [ss_sold_time_sk#15, ss_store_sk#17, t_time_sk#20] +Arguments: [ss_store_sk#17], [ss_store_sk#17] -(42) Project [codegen id : 8] -Output [1]: [ss_store_sk#19] -Input [3]: [ss_sold_time_sk#17, ss_store_sk#19, t_time_sk#22] +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#23] -(43) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#25] - -(44) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#19] -Right keys [1]: [s_store_sk#25] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#17] +Right output [1]: [s_store_sk#23] +Arguments: [ss_store_sk#17], [s_store_sk#23], Inner, BuildRight -(45) Project [codegen id : 8] -Output: [] -Input [2]: [ss_store_sk#19, s_store_sk#25] +(40) CometProject +Input [2]: [ss_store_sk#17, s_store_sk#23] -(46) HashAggregate [codegen id : 8] +(41) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#26] -Results [1]: [count#27] -(47) Exchange -Input [1]: [count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(42) CometExchange +Input [1]: [count#24] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(48) HashAggregate [codegen id : 9] -Input [1]: [count#27] +(43) CometHashAggregate +Input [1]: [count#24] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#28] -Results [1]: [count(1)#28 AS h9_to_9_30#29] -(49) BroadcastExchange -Input [1]: [h9_to_9_30#29] -Arguments: IdentityBroadcastMode, [plan_id=7] +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [h9_to_9_30#25] + +(45) BroadcastExchange +Input [1]: [h9_to_9_30#25] +Arguments: IdentityBroadcastMode, [plan_id=3] -(50) BroadcastNestedLoopJoin [codegen id : 40] +(46) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(51) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] +(47) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 13] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] +(48) CometFilter +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_hdemo_sk#27) AND isnotnull(ss_sold_time_sk#26)) AND isnotnull(ss_store_sk#28)) -(53) Filter [codegen id : 13] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] -Condition : ((isnotnull(ss_hdemo_sk#31) AND isnotnull(ss_sold_time_sk#30)) AND isnotnull(ss_store_sk#32)) +(49) CometProject +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28], [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] -(54) Project [codegen id : 13] -Output [3]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] +(50) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#30] -(55) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#34] - -(56) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_hdemo_sk#31] -Right keys [1]: [hd_demo_sk#34] -Join type: Inner -Join condition: None +(51) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] +Right output [1]: [hd_demo_sk#30] +Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight -(57) Project [codegen id : 13] -Output [2]: [ss_sold_time_sk#30, ss_store_sk#32] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, hd_demo_sk#34] +(52) CometProject +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] +Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(58) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#35, t_hour#36, t_minute#37] +(53) CometNativeScan parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 11] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] +(54) CometFilter +Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] +Condition : ((((isnotnull(t_hour#32) AND isnotnull(t_minute#33)) AND (t_hour#32 = 9)) AND (t_minute#33 >= 30)) AND isnotnull(t_time_sk#31)) -(60) Filter [codegen id : 11] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Condition : ((((isnotnull(t_hour#36) AND isnotnull(t_minute#37)) AND (t_hour#36 = 9)) AND (t_minute#37 >= 30)) AND isnotnull(t_time_sk#35)) +(55) CometProject +Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] +Arguments: [t_time_sk#31], [t_time_sk#31] -(61) Project [codegen id : 11] -Output [1]: [t_time_sk#35] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] +(56) CometBroadcastExchange +Input [1]: [t_time_sk#31] +Arguments: [t_time_sk#31] -(62) BroadcastExchange -Input [1]: [t_time_sk#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(57) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#26, ss_store_sk#28] +Right output [1]: [t_time_sk#31] +Arguments: [ss_sold_time_sk#26], [t_time_sk#31], Inner, BuildRight -(63) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_sold_time_sk#30] -Right keys [1]: [t_time_sk#35] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 13] -Output [1]: [ss_store_sk#32] -Input [3]: [ss_sold_time_sk#30, ss_store_sk#32, t_time_sk#35] +(58) CometProject +Input [3]: [ss_sold_time_sk#26, ss_store_sk#28, t_time_sk#31] +Arguments: [ss_store_sk#28], [ss_store_sk#28] -(65) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#38] +(59) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#34] -(66) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_store_sk#32] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None +(60) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#28] +Right output [1]: [s_store_sk#34] +Arguments: [ss_store_sk#28], [s_store_sk#34], Inner, BuildRight -(67) Project [codegen id : 13] -Output: [] -Input [2]: [ss_store_sk#32, s_store_sk#38] +(61) CometProject +Input [2]: [ss_store_sk#28, s_store_sk#34] -(68) HashAggregate [codegen id : 13] +(62) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#39] -Results [1]: [count#40] -(69) Exchange -Input [1]: [count#40] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] +(63) CometExchange +Input [1]: [count#35] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(70) HashAggregate [codegen id : 14] -Input [1]: [count#40] +(64) CometHashAggregate +Input [1]: [count#35] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#41] -Results [1]: [count(1)#41 AS h9_30_to_10#42] -(71) BroadcastExchange -Input [1]: [h9_30_to_10#42] -Arguments: IdentityBroadcastMode, [plan_id=10] +(65) CometColumnarToRow [codegen id : 2] +Input [1]: [h9_30_to_10#36] -(72) BroadcastNestedLoopJoin [codegen id : 40] +(66) BroadcastExchange +Input [1]: [h9_30_to_10#36] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(67) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(73) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] +(68) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(74) ColumnarToRow [codegen id : 18] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] - -(75) Filter [codegen id : 18] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] -Condition : ((isnotnull(ss_hdemo_sk#44) AND isnotnull(ss_sold_time_sk#43)) AND isnotnull(ss_store_sk#45)) +(69) CometFilter +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Condition : ((isnotnull(ss_hdemo_sk#38) AND isnotnull(ss_sold_time_sk#37)) AND isnotnull(ss_store_sk#39)) -(76) Project [codegen id : 18] -Output [3]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] +(70) CometProject +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39], [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] -(77) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#47] +(71) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#41] -(78) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_hdemo_sk#44] -Right keys [1]: [hd_demo_sk#47] -Join type: Inner -Join condition: None +(72) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] +Right output [1]: [hd_demo_sk#41] +Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight -(79) Project [codegen id : 18] -Output [2]: [ss_sold_time_sk#43, ss_store_sk#45] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, hd_demo_sk#47] +(73) CometProject +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] +Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(80) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#48, t_hour#49, t_minute#50] +(74) CometNativeScan parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(81) ColumnarToRow [codegen id : 16] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] +(75) CometFilter +Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] +Condition : ((((isnotnull(t_hour#43) AND isnotnull(t_minute#44)) AND (t_hour#43 = 10)) AND (t_minute#44 < 30)) AND isnotnull(t_time_sk#42)) -(82) Filter [codegen id : 16] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] -Condition : ((((isnotnull(t_hour#49) AND isnotnull(t_minute#50)) AND (t_hour#49 = 10)) AND (t_minute#50 < 30)) AND isnotnull(t_time_sk#48)) +(76) CometProject +Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] +Arguments: [t_time_sk#42], [t_time_sk#42] -(83) Project [codegen id : 16] -Output [1]: [t_time_sk#48] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] +(77) CometBroadcastExchange +Input [1]: [t_time_sk#42] +Arguments: [t_time_sk#42] -(84) BroadcastExchange -Input [1]: [t_time_sk#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] +(78) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#37, ss_store_sk#39] +Right output [1]: [t_time_sk#42] +Arguments: [ss_sold_time_sk#37], [t_time_sk#42], Inner, BuildRight -(85) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_sold_time_sk#43] -Right keys [1]: [t_time_sk#48] -Join type: Inner -Join condition: None +(79) CometProject +Input [3]: [ss_sold_time_sk#37, ss_store_sk#39, t_time_sk#42] +Arguments: [ss_store_sk#39], [ss_store_sk#39] -(86) Project [codegen id : 18] -Output [1]: [ss_store_sk#45] -Input [3]: [ss_sold_time_sk#43, ss_store_sk#45, t_time_sk#48] +(80) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#45] -(87) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#51] +(81) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#39] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#39], [s_store_sk#45], Inner, BuildRight -(88) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_store_sk#45] -Right keys [1]: [s_store_sk#51] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 18] -Output: [] -Input [2]: [ss_store_sk#45, s_store_sk#51] +(82) CometProject +Input [2]: [ss_store_sk#39, s_store_sk#45] -(90) HashAggregate [codegen id : 18] +(83) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#52] -Results [1]: [count#53] -(91) Exchange -Input [1]: [count#53] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(84) CometExchange +Input [1]: [count#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(92) HashAggregate [codegen id : 19] -Input [1]: [count#53] +(85) CometHashAggregate +Input [1]: [count#46] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#54] -Results [1]: [count(1)#54 AS h10_to_10_30#55] -(93) BroadcastExchange -Input [1]: [h10_to_10_30#55] -Arguments: IdentityBroadcastMode, [plan_id=13] +(86) CometColumnarToRow [codegen id : 3] +Input [1]: [h10_to_10_30#47] + +(87) BroadcastExchange +Input [1]: [h10_to_10_30#47] +Arguments: IdentityBroadcastMode, [plan_id=7] -(94) BroadcastNestedLoopJoin [codegen id : 40] +(88) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(95) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] +(89) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 23] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] - -(97) Filter [codegen id : 23] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] -Condition : ((isnotnull(ss_hdemo_sk#57) AND isnotnull(ss_sold_time_sk#56)) AND isnotnull(ss_store_sk#58)) +(90) CometFilter +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Condition : ((isnotnull(ss_hdemo_sk#49) AND isnotnull(ss_sold_time_sk#48)) AND isnotnull(ss_store_sk#50)) -(98) Project [codegen id : 23] -Output [3]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] +(91) CometProject +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50], [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] -(99) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#60] +(92) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#52] -(100) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_hdemo_sk#57] -Right keys [1]: [hd_demo_sk#60] -Join type: Inner -Join condition: None +(93) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] +Right output [1]: [hd_demo_sk#52] +Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight -(101) Project [codegen id : 23] -Output [2]: [ss_sold_time_sk#56, ss_store_sk#58] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, hd_demo_sk#60] +(94) CometProject +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] +Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(102) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#61, t_hour#62, t_minute#63] +(95) CometNativeScan parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 21] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] +(96) CometFilter +Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] +Condition : ((((isnotnull(t_hour#54) AND isnotnull(t_minute#55)) AND (t_hour#54 = 10)) AND (t_minute#55 >= 30)) AND isnotnull(t_time_sk#53)) -(104) Filter [codegen id : 21] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] -Condition : ((((isnotnull(t_hour#62) AND isnotnull(t_minute#63)) AND (t_hour#62 = 10)) AND (t_minute#63 >= 30)) AND isnotnull(t_time_sk#61)) +(97) CometProject +Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] +Arguments: [t_time_sk#53], [t_time_sk#53] -(105) Project [codegen id : 21] -Output [1]: [t_time_sk#61] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] +(98) CometBroadcastExchange +Input [1]: [t_time_sk#53] +Arguments: [t_time_sk#53] -(106) BroadcastExchange -Input [1]: [t_time_sk#61] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] +(99) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#48, ss_store_sk#50] +Right output [1]: [t_time_sk#53] +Arguments: [ss_sold_time_sk#48], [t_time_sk#53], Inner, BuildRight -(107) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_sold_time_sk#56] -Right keys [1]: [t_time_sk#61] -Join type: Inner -Join condition: None +(100) CometProject +Input [3]: [ss_sold_time_sk#48, ss_store_sk#50, t_time_sk#53] +Arguments: [ss_store_sk#50], [ss_store_sk#50] -(108) Project [codegen id : 23] -Output [1]: [ss_store_sk#58] -Input [3]: [ss_sold_time_sk#56, ss_store_sk#58, t_time_sk#61] +(101) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#56] -(109) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#64] +(102) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#50] +Right output [1]: [s_store_sk#56] +Arguments: [ss_store_sk#50], [s_store_sk#56], Inner, BuildRight -(110) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_store_sk#58] -Right keys [1]: [s_store_sk#64] -Join type: Inner -Join condition: None - -(111) Project [codegen id : 23] -Output: [] -Input [2]: [ss_store_sk#58, s_store_sk#64] +(103) CometProject +Input [2]: [ss_store_sk#50, s_store_sk#56] -(112) HashAggregate [codegen id : 23] +(104) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#65] -Results [1]: [count#66] -(113) Exchange -Input [1]: [count#66] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(105) CometExchange +Input [1]: [count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(114) HashAggregate [codegen id : 24] -Input [1]: [count#66] +(106) CometHashAggregate +Input [1]: [count#57] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#67] -Results [1]: [count(1)#67 AS h10_30_to_11#68] -(115) BroadcastExchange -Input [1]: [h10_30_to_11#68] -Arguments: IdentityBroadcastMode, [plan_id=16] +(107) CometColumnarToRow [codegen id : 4] +Input [1]: [h10_30_to_11#58] -(116) BroadcastNestedLoopJoin [codegen id : 40] +(108) BroadcastExchange +Input [1]: [h10_30_to_11#58] +Arguments: IdentityBroadcastMode, [plan_id=9] + +(109) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(117) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] +(110) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(118) ColumnarToRow [codegen id : 28] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] - -(119) Filter [codegen id : 28] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] -Condition : ((isnotnull(ss_hdemo_sk#70) AND isnotnull(ss_sold_time_sk#69)) AND isnotnull(ss_store_sk#71)) +(111) CometFilter +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Condition : ((isnotnull(ss_hdemo_sk#60) AND isnotnull(ss_sold_time_sk#59)) AND isnotnull(ss_store_sk#61)) -(120) Project [codegen id : 28] -Output [3]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] +(112) CometProject +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61], [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] -(121) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#73] +(113) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#63] -(122) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_hdemo_sk#70] -Right keys [1]: [hd_demo_sk#73] -Join type: Inner -Join condition: None +(114) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] +Right output [1]: [hd_demo_sk#63] +Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight -(123) Project [codegen id : 28] -Output [2]: [ss_sold_time_sk#69, ss_store_sk#71] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, hd_demo_sk#73] +(115) CometProject +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] +Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(124) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#74, t_hour#75, t_minute#76] +(116) CometNativeScan parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 26] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] +(117) CometFilter +Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] +Condition : ((((isnotnull(t_hour#65) AND isnotnull(t_minute#66)) AND (t_hour#65 = 11)) AND (t_minute#66 < 30)) AND isnotnull(t_time_sk#64)) -(126) Filter [codegen id : 26] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] -Condition : ((((isnotnull(t_hour#75) AND isnotnull(t_minute#76)) AND (t_hour#75 = 11)) AND (t_minute#76 < 30)) AND isnotnull(t_time_sk#74)) +(118) CometProject +Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] +Arguments: [t_time_sk#64], [t_time_sk#64] -(127) Project [codegen id : 26] -Output [1]: [t_time_sk#74] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] +(119) CometBroadcastExchange +Input [1]: [t_time_sk#64] +Arguments: [t_time_sk#64] -(128) BroadcastExchange -Input [1]: [t_time_sk#74] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] +(120) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#59, ss_store_sk#61] +Right output [1]: [t_time_sk#64] +Arguments: [ss_sold_time_sk#59], [t_time_sk#64], Inner, BuildRight -(129) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_sold_time_sk#69] -Right keys [1]: [t_time_sk#74] -Join type: Inner -Join condition: None +(121) CometProject +Input [3]: [ss_sold_time_sk#59, ss_store_sk#61, t_time_sk#64] +Arguments: [ss_store_sk#61], [ss_store_sk#61] -(130) Project [codegen id : 28] -Output [1]: [ss_store_sk#71] -Input [3]: [ss_sold_time_sk#69, ss_store_sk#71, t_time_sk#74] +(122) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#67] -(131) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#77] +(123) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#61] +Right output [1]: [s_store_sk#67] +Arguments: [ss_store_sk#61], [s_store_sk#67], Inner, BuildRight -(132) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_store_sk#71] -Right keys [1]: [s_store_sk#77] -Join type: Inner -Join condition: None +(124) CometProject +Input [2]: [ss_store_sk#61, s_store_sk#67] -(133) Project [codegen id : 28] -Output: [] -Input [2]: [ss_store_sk#71, s_store_sk#77] - -(134) HashAggregate [codegen id : 28] +(125) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#78] -Results [1]: [count#79] -(135) Exchange -Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] +(126) CometExchange +Input [1]: [count#68] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(136) HashAggregate [codegen id : 29] -Input [1]: [count#79] +(127) CometHashAggregate +Input [1]: [count#68] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#80] -Results [1]: [count(1)#80 AS h11_to_11_30#81] -(137) BroadcastExchange -Input [1]: [h11_to_11_30#81] -Arguments: IdentityBroadcastMode, [plan_id=19] +(128) CometColumnarToRow [codegen id : 5] +Input [1]: [h11_to_11_30#69] -(138) BroadcastNestedLoopJoin [codegen id : 40] +(129) BroadcastExchange +Input [1]: [h11_to_11_30#69] +Arguments: IdentityBroadcastMode, [plan_id=11] + +(130) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(139) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] +(131) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(140) ColumnarToRow [codegen id : 33] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] - -(141) Filter [codegen id : 33] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] -Condition : ((isnotnull(ss_hdemo_sk#83) AND isnotnull(ss_sold_time_sk#82)) AND isnotnull(ss_store_sk#84)) +(132) CometFilter +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Condition : ((isnotnull(ss_hdemo_sk#71) AND isnotnull(ss_sold_time_sk#70)) AND isnotnull(ss_store_sk#72)) -(142) Project [codegen id : 33] -Output [3]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] +(133) CometProject +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72], [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] -(143) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#86] +(134) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#74] -(144) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_hdemo_sk#83] -Right keys [1]: [hd_demo_sk#86] -Join type: Inner -Join condition: None +(135) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] +Right output [1]: [hd_demo_sk#74] +Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight -(145) Project [codegen id : 33] -Output [2]: [ss_sold_time_sk#82, ss_store_sk#84] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, hd_demo_sk#86] +(136) CometProject +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] +Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(146) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#87, t_hour#88, t_minute#89] +(137) CometNativeScan parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(147) ColumnarToRow [codegen id : 31] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] +(138) CometFilter +Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] +Condition : ((((isnotnull(t_hour#76) AND isnotnull(t_minute#77)) AND (t_hour#76 = 11)) AND (t_minute#77 >= 30)) AND isnotnull(t_time_sk#75)) -(148) Filter [codegen id : 31] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] -Condition : ((((isnotnull(t_hour#88) AND isnotnull(t_minute#89)) AND (t_hour#88 = 11)) AND (t_minute#89 >= 30)) AND isnotnull(t_time_sk#87)) +(139) CometProject +Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] +Arguments: [t_time_sk#75], [t_time_sk#75] -(149) Project [codegen id : 31] -Output [1]: [t_time_sk#87] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] +(140) CometBroadcastExchange +Input [1]: [t_time_sk#75] +Arguments: [t_time_sk#75] -(150) BroadcastExchange -Input [1]: [t_time_sk#87] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] +(141) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#70, ss_store_sk#72] +Right output [1]: [t_time_sk#75] +Arguments: [ss_sold_time_sk#70], [t_time_sk#75], Inner, BuildRight -(151) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_sold_time_sk#82] -Right keys [1]: [t_time_sk#87] -Join type: Inner -Join condition: None +(142) CometProject +Input [3]: [ss_sold_time_sk#70, ss_store_sk#72, t_time_sk#75] +Arguments: [ss_store_sk#72], [ss_store_sk#72] -(152) Project [codegen id : 33] -Output [1]: [ss_store_sk#84] -Input [3]: [ss_sold_time_sk#82, ss_store_sk#84, t_time_sk#87] +(143) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#78] -(153) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#90] +(144) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#72] +Right output [1]: [s_store_sk#78] +Arguments: [ss_store_sk#72], [s_store_sk#78], Inner, BuildRight -(154) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_store_sk#84] -Right keys [1]: [s_store_sk#90] -Join type: Inner -Join condition: None - -(155) Project [codegen id : 33] -Output: [] -Input [2]: [ss_store_sk#84, s_store_sk#90] +(145) CometProject +Input [2]: [ss_store_sk#72, s_store_sk#78] -(156) HashAggregate [codegen id : 33] +(146) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#91] -Results [1]: [count#92] -(157) Exchange -Input [1]: [count#92] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21] +(147) CometExchange +Input [1]: [count#79] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(158) HashAggregate [codegen id : 34] -Input [1]: [count#92] +(148) CometHashAggregate +Input [1]: [count#79] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#93] -Results [1]: [count(1)#93 AS h11_30_to_12#94] -(159) BroadcastExchange -Input [1]: [h11_30_to_12#94] -Arguments: IdentityBroadcastMode, [plan_id=22] +(149) CometColumnarToRow [codegen id : 6] +Input [1]: [h11_30_to_12#80] + +(150) BroadcastExchange +Input [1]: [h11_30_to_12#80] +Arguments: IdentityBroadcastMode, [plan_id=13] -(160) BroadcastNestedLoopJoin [codegen id : 40] +(151) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(161) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] +(152) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(162) ColumnarToRow [codegen id : 38] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] - -(163) Filter [codegen id : 38] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] -Condition : ((isnotnull(ss_hdemo_sk#96) AND isnotnull(ss_sold_time_sk#95)) AND isnotnull(ss_store_sk#97)) +(153) CometFilter +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Condition : ((isnotnull(ss_hdemo_sk#82) AND isnotnull(ss_sold_time_sk#81)) AND isnotnull(ss_store_sk#83)) -(164) Project [codegen id : 38] -Output [3]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] +(154) CometProject +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83], [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] -(165) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#99] +(155) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#85] -(166) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_hdemo_sk#96] -Right keys [1]: [hd_demo_sk#99] -Join type: Inner -Join condition: None +(156) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] +Right output [1]: [hd_demo_sk#85] +Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight -(167) Project [codegen id : 38] -Output [2]: [ss_sold_time_sk#95, ss_store_sk#97] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, hd_demo_sk#99] +(157) CometProject +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] +Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(168) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#100, t_hour#101, t_minute#102] +(158) CometNativeScan parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(169) ColumnarToRow [codegen id : 36] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] +(159) CometFilter +Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] +Condition : ((((isnotnull(t_hour#87) AND isnotnull(t_minute#88)) AND (t_hour#87 = 12)) AND (t_minute#88 < 30)) AND isnotnull(t_time_sk#86)) -(170) Filter [codegen id : 36] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] -Condition : ((((isnotnull(t_hour#101) AND isnotnull(t_minute#102)) AND (t_hour#101 = 12)) AND (t_minute#102 < 30)) AND isnotnull(t_time_sk#100)) +(160) CometProject +Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] +Arguments: [t_time_sk#86], [t_time_sk#86] -(171) Project [codegen id : 36] -Output [1]: [t_time_sk#100] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] +(161) CometBroadcastExchange +Input [1]: [t_time_sk#86] +Arguments: [t_time_sk#86] -(172) BroadcastExchange -Input [1]: [t_time_sk#100] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=23] +(162) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#81, ss_store_sk#83] +Right output [1]: [t_time_sk#86] +Arguments: [ss_sold_time_sk#81], [t_time_sk#86], Inner, BuildRight -(173) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_sold_time_sk#95] -Right keys [1]: [t_time_sk#100] -Join type: Inner -Join condition: None +(163) CometProject +Input [3]: [ss_sold_time_sk#81, ss_store_sk#83, t_time_sk#86] +Arguments: [ss_store_sk#83], [ss_store_sk#83] -(174) Project [codegen id : 38] -Output [1]: [ss_store_sk#97] -Input [3]: [ss_sold_time_sk#95, ss_store_sk#97, t_time_sk#100] +(164) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#89] -(175) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#103] +(165) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#83] +Right output [1]: [s_store_sk#89] +Arguments: [ss_store_sk#83], [s_store_sk#89], Inner, BuildRight -(176) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_store_sk#97] -Right keys [1]: [s_store_sk#103] -Join type: Inner -Join condition: None +(166) CometProject +Input [2]: [ss_store_sk#83, s_store_sk#89] -(177) Project [codegen id : 38] -Output: [] -Input [2]: [ss_store_sk#97, s_store_sk#103] - -(178) HashAggregate [codegen id : 38] +(167) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#104] -Results [1]: [count#105] -(179) Exchange -Input [1]: [count#105] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=24] +(168) CometExchange +Input [1]: [count#90] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(180) HashAggregate [codegen id : 39] -Input [1]: [count#105] +(169) CometHashAggregate +Input [1]: [count#90] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#106] -Results [1]: [count(1)#106 AS h12_to_12_30#107] -(181) BroadcastExchange -Input [1]: [h12_to_12_30#107] -Arguments: IdentityBroadcastMode, [plan_id=25] +(170) CometColumnarToRow [codegen id : 7] +Input [1]: [h12_to_12_30#91] + +(171) BroadcastExchange +Input [1]: [h12_to_12_30#91] +Arguments: IdentityBroadcastMode, [plan_id=15] -(182) BroadcastNestedLoopJoin [codegen id : 40] +(172) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/extended.txt new file mode 100644 index 0000000000..779053cb5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/extended.txt @@ -0,0 +1,216 @@ +BroadcastNestedLoopJoin +:- BroadcastNestedLoopJoin +: :- BroadcastNestedLoopJoin +: : :- BroadcastNestedLoopJoin +: : : :- BroadcastNestedLoopJoin +: : : : :- BroadcastNestedLoopJoin +: : : : : :- BroadcastNestedLoopJoin +: : : : : : :- CometColumnarToRow +: : : : : : : +- CometHashAggregate +: : : : : : : +- CometExchange +: : : : : : : +- CometHashAggregate +: : : : : : : +- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : :- CometProject +: : : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : : :- CometProject +: : : : : : : : : : +- CometFilter +: : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : : : +- CometBroadcastExchange +: : : : : : : : : +- CometProject +: : : : : : : : : +- CometFilter +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : : : +- CometBroadcastExchange +: : : : : : : : +- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : : : +- BroadcastExchange +: : : : : : +- CometColumnarToRow +: : : : : : +- CometHashAggregate +: : : : : : +- CometExchange +: : : : : : +- CometHashAggregate +: : : : : : +- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometBroadcastHashJoin +: : : : : : : : :- CometProject +: : : : : : : : : +- CometFilter +: : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : : +- CometBroadcastExchange +: : : : : : : : +- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : : +- BroadcastExchange +: : : : : +- CometColumnarToRow +: : : : : +- CometHashAggregate +: : : : : +- CometExchange +: : : : : +- CometHashAggregate +: : : : : +- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometBroadcastHashJoin +: : : : : : : :- CometProject +: : : : : : : : +- CometFilter +: : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : : +- CometBroadcastExchange +: : : : : : : +- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store +: : : : +- BroadcastExchange +: : : : +- CometColumnarToRow +: : : : +- CometHashAggregate +: : : : +- CometExchange +: : : : +- CometHashAggregate +: : : : +- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometBroadcastHashJoin +: : : : : : :- CometProject +: : : : : : : +- CometFilter +: : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : : +- CometBroadcastExchange +: : : : : : +- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store +: : : +- BroadcastExchange +: : : +- CometColumnarToRow +: : : +- CometHashAggregate +: : : +- CometExchange +: : : +- CometHashAggregate +: : : +- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometBroadcastHashJoin +: : : : : :- CometProject +: : : : : : +- CometFilter +: : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : : +- CometBroadcastExchange +: : : : : +- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.store +: : +- BroadcastExchange +: : +- CometColumnarToRow +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometBroadcastHashJoin +: : : : :- CometProject +: : : : : +- CometFilter +: : : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : : +- CometBroadcastExchange +: : : : +- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.time_dim +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store +: +- BroadcastExchange +: +- CometColumnarToRow +: +- CometHashAggregate +: +- CometExchange +: +- CometHashAggregate +: +- CometProject +: +- CometBroadcastHashJoin +: :- CometProject +: : +- CometBroadcastHashJoin +: : :- CometProject +: : : +- CometBroadcastHashJoin +: : : :- CometProject +: : : : +- CometFilter +: : : : +- CometNativeScan parquet spark_catalog.default.store_sales +: : : +- CometBroadcastExchange +: : : +- CometProject +: : : +- CometFilter +: : : +- CometNativeScan parquet spark_catalog.default.household_demographics +: : +- CometBroadcastExchange +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.time_dim +: +- CometBroadcastExchange +: +- CometProject +: +- CometFilter +: +- CometNativeScan parquet spark_catalog.default.store ++- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 192 out of 206 eligible operators (93%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/simplified.txt index 12778886b5..cef8e0e760 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/simplified.txt @@ -1,4 +1,4 @@ -WholeStageCodegen (40) +WholeStageCodegen (8) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin @@ -6,260 +6,190 @@ WholeStageCodegen (40) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - HashAggregate [count] [count(1),h8_30_to_9,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometHashAggregate [count] [h8_30_to_9,count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 - WholeStageCodegen (9) - HashAggregate [count] [count(1),h9_to_9_30,count] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #6 - WholeStageCodegen (8) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h9_to_9_30,count(1)] + CometExchange #6 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #7 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 - WholeStageCodegen (14) - HashAggregate [count] [count(1),h9_30_to_10,count] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (11) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h9_30_to_10,count(1)] + CometExchange #9 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #10 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 - WholeStageCodegen (19) - HashAggregate [count] [count(1),h10_to_10_30,count] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange #12 - WholeStageCodegen (18) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (16) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h10_to_10_30,count(1)] + CometExchange #12 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #13 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 - WholeStageCodegen (24) - HashAggregate [count] [count(1),h10_30_to_11,count] + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Exchange #15 - WholeStageCodegen (23) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (21) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h10_30_to_11,count(1)] + CometExchange #15 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #16 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 - WholeStageCodegen (29) - HashAggregate [count] [count(1),h11_to_11_30,count] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange #18 - WholeStageCodegen (28) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (26) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h11_to_11_30,count(1)] + CometExchange #18 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #19 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 - WholeStageCodegen (34) - HashAggregate [count] [count(1),h11_30_to_12,count] + WholeStageCodegen (6) + CometColumnarToRow InputAdapter - Exchange #21 - WholeStageCodegen (33) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #22 - WholeStageCodegen (31) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h11_30_to_12,count(1)] + CometExchange #21 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #22 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 - WholeStageCodegen (39) - HashAggregate [count] [count(1),h12_to_12_30,count] + WholeStageCodegen (7) + CometColumnarToRow InputAdapter - Exchange #24 - WholeStageCodegen (38) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #25 - WholeStageCodegen (36) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h12_to_12_30,count(1)] + CometExchange #24 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #25 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/explain.txt index f984b6c76b..22dd1d2dd8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/explain.txt @@ -1,179 +1,165 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Project (30) - +- * Filter (29) - +- Window (28) - +- * Sort (27) - +- Exchange (26) - +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.store_sales (4) - : +- BroadcastExchange (14) - : +- * Project (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.date_dim (10) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.store (17) - - -(1) Scan parquet spark_catalog.default.item +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometNativeScan parquet spark_catalog.default.store (15) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(In(i_category, [Books ,Electronics ,Sports ]),In(i_class, [computers ,football ,stereo ])),And(In(i_category, [Jewelry ,Men ,Women ]),In(i_class, [birdal ,dresses ,shirts ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Electronics ,Sports ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (computers ,stereo ,football )) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Men ,Jewelry ,Women ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) -(3) Filter [codegen id : 4] +(3) CometProject Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) +Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) AS i_class#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) AS i_category#7] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +(5) CometFilter +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Condition : (isnotnull(ss_item_sk#8) AND isnotnull(ss_store_sk#9)) -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(7) BroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7] +Right output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(9) Project [codegen id : 4] -Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +(10) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) -(13) Project [codegen id : 2] -Output [2]: [d_date_sk#9, d_moy#11] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +(11) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] -(14) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight -(16) Project [codegen id : 4] -Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] +(14) CometProject +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] -(17) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] +(15) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(19) Filter [codegen id : 3] -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) - -(20) BroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] - -(23) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum#15] -Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] - -(24) Exchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#17] -Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS _w0#19] - -(26) Exchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(27) Sort [codegen id : 6] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST], false, 0 - -(28) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] - -(29) Filter [codegen id : 7] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] -Condition : CASE WHEN NOT (avg_monthly_sales#20 = 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END - -(30) Project [codegen id : 7] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] - -(31) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] +(16) CometFilter +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) + +(17) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] + +(18) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight + +(19) CometProject +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] + +(20) CometHashAggregate +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] + +(21) CometExchange +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometHashAggregate +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] + +(23) CometExchange +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] + +(26) Window +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] + +(27) Filter [codegen id : 2] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN NOT (avg_monthly_sales#21 = 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END + +(28) Project [codegen id : 2] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] + +(29) TakeOrderedAndProject +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [(sum_sales#19 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/extended.txt new file mode 100644 index 0000000000..44fd48691b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/extended.txt @@ -0,0 +1,31 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Filter + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 28 eligible operators (85%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/simplified.txt index 8ea1fd5d3d..56394060ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/simplified.txt @@ -1,48 +1,33 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales] Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_moy] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometHashAggregate [sum] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] + CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] + CometFilter [i_item_sk,i_brand,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/explain.txt index e32db67408..da194f2825 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/explain.txt @@ -1,23 +1,23 @@ == Physical Plan == * Project (4) -+- * Filter (3) - +- * ColumnarToRow (2) - +- Scan parquet spark_catalog.default.reason (1) ++- * CometColumnarToRow (3) + +- CometFilter (2) + +- CometNativeScan parquet spark_catalog.default.reason (1) -(1) Scan parquet spark_catalog.default.reason +(1) CometNativeScan parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [1]: [r_reason_sk#1] +Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) -(3) Filter [codegen id : 1] +(3) CometColumnarToRow [codegen id : 1] Input [1]: [r_reason_sk#1] -Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) (4) Project [codegen id : 1] Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] @@ -26,275 +26,255 @@ Input [1]: [r_reason_sk#1] ===== Subqueries ===== Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] -* Project (12) -+- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) - +- Scan parquet spark_catalog.default.store_sales (5) +* CometColumnarToRow (12) ++- CometProject (11) + +- CometHashAggregate (10) + +- CometExchange (9) + +- CometHashAggregate (8) + +- CometProject (7) + +- CometFilter (6) + +- CometNativeScan parquet spark_catalog.default.store_sales (5) -(5) Scan parquet spark_catalog.default.store_sales +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quantity#17 <= 20)) -(8) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#18, ss_net_paid#19] +(7) CometProject Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Arguments: [ss_ext_discount_amt#18, ss_net_paid#19], [ss_ext_discount_amt#18, ss_net_paid#19] -(9) HashAggregate [codegen id : 1] +(8) CometHashAggregate Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [5]: [count#21, sum#22, count#23, sum#24, count#25] -Results [5]: [count#26, sum#27, count#28, sum#29, count#30] -(10) Exchange -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] +(9) CometExchange +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(11) HashAggregate [codegen id : 2] -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] +(10) CometHashAggregate +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [3]: [count(1)#31, avg(UnscaledValue(ss_ext_discount_amt#18))#32, avg(UnscaledValue(ss_net_paid#19))#33] -Results [3]: [count(1)#31 AS count(1)#34, cast((avg(UnscaledValue(ss_ext_discount_amt#18))#32 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#35, cast((avg(UnscaledValue(ss_net_paid#19))#33 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#36] -(12) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#34, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#35, avg(ss_net_paid), avg(ss_net_paid)#36) AS mergedValue#37] -Input [3]: [count(1)#34, avg(ss_ext_discount_amt)#35, avg(ss_net_paid)#36] +(11) CometProject +Input [3]: [count(1)#26, avg(ss_ext_discount_amt)#27, avg(ss_net_paid)#28] +Arguments: [mergedValue#29], [named_struct(count(1), count(1)#26, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#27, avg(ss_net_paid), avg(ss_net_paid)#28) AS mergedValue#29] + +(12) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#29] Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] -* Project (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * Filter (15) - +- * ColumnarToRow (14) - +- Scan parquet spark_catalog.default.store_sales (13) - - -(13) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] +* CometColumnarToRow (20) ++- CometProject (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometProject (15) + +- CometFilter (14) + +- CometNativeScan parquet spark_catalog.default.store_sales (13) + + +(13) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] +(14) CometFilter +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Condition : ((isnotnull(ss_quantity#30) AND (ss_quantity#30 >= 21)) AND (ss_quantity#30 <= 40)) -(15) Filter [codegen id : 1] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Condition : ((isnotnull(ss_quantity#38) AND (ss_quantity#38 >= 21)) AND (ss_quantity#38 <= 40)) +(15) CometProject +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Arguments: [ss_ext_discount_amt#31, ss_net_paid#32], [ss_ext_discount_amt#31, ss_net_paid#32] -(16) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#39, ss_net_paid#40] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] - -(17) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] +(16) CometHashAggregate +Input [2]: [ss_ext_discount_amt#31, ss_net_paid#32] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#39)), partial_avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [5]: [count#42, sum#43, count#44, sum#45, count#46] -Results [5]: [count#47, sum#48, count#49, sum#50, count#51] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] -(18) Exchange -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(17) CometExchange +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(19) HashAggregate [codegen id : 2] -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +(18) CometHashAggregate +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#39)), avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [3]: [count(1)#52, avg(UnscaledValue(ss_ext_discount_amt#39))#53, avg(UnscaledValue(ss_net_paid#40))#54] -Results [3]: [count(1)#52 AS count(1)#55, cast((avg(UnscaledValue(ss_ext_discount_amt#39))#53 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#56, cast((avg(UnscaledValue(ss_net_paid#40))#54 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#57] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(UnscaledValue(ss_net_paid#32))] + +(19) CometProject +Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] +Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] -(20) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#55, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#56, avg(ss_net_paid), avg(ss_net_paid)#57) AS mergedValue#58] -Input [3]: [count(1)#55, avg(ss_ext_discount_amt)#56, avg(ss_net_paid)#57] +(20) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#42] Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] -* Project (28) -+- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) - +- Scan parquet spark_catalog.default.store_sales (21) - - -(21) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] +* CometColumnarToRow (28) ++- CometProject (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometFilter (22) + +- CometNativeScan parquet spark_catalog.default.store_sales (21) + + +(21) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] +(22) CometFilter +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Condition : ((isnotnull(ss_quantity#43) AND (ss_quantity#43 >= 41)) AND (ss_quantity#43 <= 60)) -(23) Filter [codegen id : 1] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Condition : ((isnotnull(ss_quantity#59) AND (ss_quantity#59 >= 41)) AND (ss_quantity#59 <= 60)) +(23) CometProject +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Arguments: [ss_ext_discount_amt#44, ss_net_paid#45], [ss_ext_discount_amt#44, ss_net_paid#45] -(24) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#60, ss_net_paid#61] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] - -(25) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] +(24) CometHashAggregate +Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#60)), partial_avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [5]: [count#63, sum#64, count#65, sum#66, count#67] -Results [5]: [count#68, sum#69, count#70, sum#71, count#72] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] -(26) Exchange -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +(25) CometExchange +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(27) HashAggregate [codegen id : 2] -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] +(26) CometHashAggregate +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#60)), avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [3]: [count(1)#73, avg(UnscaledValue(ss_ext_discount_amt#60))#74, avg(UnscaledValue(ss_net_paid#61))#75] -Results [3]: [count(1)#73 AS count(1)#76, cast((avg(UnscaledValue(ss_ext_discount_amt#60))#74 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#77, cast((avg(UnscaledValue(ss_net_paid#61))#75 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#78] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(UnscaledValue(ss_net_paid#45))] + +(27) CometProject +Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] +Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] -(28) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#76, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#77, avg(ss_net_paid), avg(ss_net_paid)#78) AS mergedValue#79] -Input [3]: [count(1)#76, avg(ss_ext_discount_amt)#77, avg(ss_net_paid)#78] +(28) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#55] Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* Project (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.store_sales (29) - - -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] +* CometColumnarToRow (36) ++- CometProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometFilter (30) + +- CometNativeScan parquet spark_catalog.default.store_sales (29) + + +(29) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] +(30) CometFilter +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Condition : ((isnotnull(ss_quantity#56) AND (ss_quantity#56 >= 61)) AND (ss_quantity#56 <= 80)) -(31) Filter [codegen id : 1] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Condition : ((isnotnull(ss_quantity#80) AND (ss_quantity#80 >= 61)) AND (ss_quantity#80 <= 80)) +(31) CometProject +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Arguments: [ss_ext_discount_amt#57, ss_net_paid#58], [ss_ext_discount_amt#57, ss_net_paid#58] -(32) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#81, ss_net_paid#82] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] - -(33) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] +(32) CometHashAggregate +Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#81)), partial_avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [5]: [count#84, sum#85, count#86, sum#87, count#88] -Results [5]: [count#89, sum#90, count#91, sum#92, count#93] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] -(34) Exchange -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(33) CometExchange +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(35) HashAggregate [codegen id : 2] -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] +(34) CometHashAggregate +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#81)), avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [3]: [count(1)#94, avg(UnscaledValue(ss_ext_discount_amt#81))#95, avg(UnscaledValue(ss_net_paid#82))#96] -Results [3]: [count(1)#94 AS count(1)#97, cast((avg(UnscaledValue(ss_ext_discount_amt#81))#95 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#98, cast((avg(UnscaledValue(ss_net_paid#82))#96 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#99] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(UnscaledValue(ss_net_paid#58))] + +(35) CometProject +Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] +Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] -(36) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#97, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#98, avg(ss_net_paid), avg(ss_net_paid)#99) AS mergedValue#100] -Input [3]: [count(1)#97, avg(ss_ext_discount_amt)#98, avg(ss_net_paid)#99] +(36) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#68] Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] -* Project (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.store_sales (37) - - -(37) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] +* CometColumnarToRow (44) ++- CometProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometFilter (38) + +- CometNativeScan parquet spark_catalog.default.store_sales (37) + + +(37) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] +(38) CometFilter +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Condition : ((isnotnull(ss_quantity#69) AND (ss_quantity#69 >= 81)) AND (ss_quantity#69 <= 100)) -(39) Filter [codegen id : 1] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Condition : ((isnotnull(ss_quantity#101) AND (ss_quantity#101 >= 81)) AND (ss_quantity#101 <= 100)) +(39) CometProject +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Arguments: [ss_ext_discount_amt#70, ss_net_paid#71], [ss_ext_discount_amt#70, ss_net_paid#71] -(40) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#102, ss_net_paid#103] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] - -(41) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] +(40) CometHashAggregate +Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#102)), partial_avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [5]: [count#105, sum#106, count#107, sum#108, count#109] -Results [5]: [count#110, sum#111, count#112, sum#113, count#114] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] -(42) Exchange -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(41) CometExchange +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(43) HashAggregate [codegen id : 2] -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] +(42) CometHashAggregate +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#102)), avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [3]: [count(1)#115, avg(UnscaledValue(ss_ext_discount_amt#102))#116, avg(UnscaledValue(ss_net_paid#103))#117] -Results [3]: [count(1)#115 AS count(1)#118, cast((avg(UnscaledValue(ss_ext_discount_amt#102))#116 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#119, cast((avg(UnscaledValue(ss_net_paid#103))#117 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#120] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(UnscaledValue(ss_net_paid#71))] + +(43) CometProject +Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] +Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] -(44) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#118, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#119, avg(ss_net_paid), avg(ss_net_paid)#120) AS mergedValue#121] -Input [3]: [count(1)#118, avg(ss_ext_discount_amt)#119, avg(ss_net_paid)#120] +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#81] Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/extended.txt new file mode 100644 index 0000000000..53ba3252cf --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/extended.txt @@ -0,0 +1,61 @@ + Project [COMET: ] +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: :- ReusedSubquery +: :- Subquery +: : +- CometColumnarToRow +: : +- CometProject +: : +- CometHashAggregate +: : +- CometExchange +: : +- CometHashAggregate +: : +- CometProject +: : +- CometFilter +: : +- CometNativeScan parquet spark_catalog.default.store_sales +: :- ReusedSubquery +: +- ReusedSubquery ++- CometColumnarToRow + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/simplified.txt index 817fb0007f..9593d6ebd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/simplified.txt @@ -1,81 +1,71 @@ WholeStageCodegen (1) Project Subquery #1 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #1 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #2 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #2 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #3 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #3 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #4 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #4 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #5 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #5 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 - Filter [r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk] + CometColumnarToRow + InputAdapter + CometFilter [r_reason_sk] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/explain.txt index 7f295e73ec..73b54f439b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/explain.txt @@ -1,292 +1,260 @@ == Physical Plan == -* Project (51) -+- * BroadcastNestedLoopJoin Inner BuildRight (50) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (18) - : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * Project (11) - : : : +- * BroadcastHashJoin Inner BuildRight (10) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (9) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) - : : : +- Scan parquet spark_catalog.default.household_demographics (5) - : : +- BroadcastExchange (16) - : : +- * Project (15) - : : +- * Filter (14) - : : +- * ColumnarToRow (13) - : : +- Scan parquet spark_catalog.default.time_dim (12) - : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) - : +- Scan parquet spark_catalog.default.web_page (19) - +- BroadcastExchange (49) - +- * HashAggregate (48) - +- Exchange (47) - +- * HashAggregate (46) - +- * Project (45) - +- * BroadcastHashJoin Inner BuildRight (44) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.web_sales (29) - : : +- ReusedExchange (33) - : +- BroadcastExchange (40) - : +- * Project (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) - : +- Scan parquet spark_catalog.default.time_dim (36) - +- ReusedExchange (43) - - -(1) Scan parquet spark_catalog.default.web_sales +* Project (47) ++- * BroadcastNestedLoopJoin Inner BuildRight (46) + :- * CometColumnarToRow (25) + : +- CometHashAggregate (24) + : +- CometExchange (23) + : +- CometHashAggregate (22) + : +- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (15) + : : +- CometBroadcastHashJoin (14) + : : :- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) + : : +- CometBroadcastExchange (13) + : : +- CometProject (12) + : : +- CometFilter (11) + : : +- CometNativeScan parquet spark_catalog.default.time_dim (10) + : +- CometBroadcastExchange (19) + : +- CometProject (18) + : +- CometFilter (17) + : +- CometNativeScan parquet spark_catalog.default.web_page (16) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (26) + : : +- ReusedExchange (29) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.time_dim (32) + +- ReusedExchange (38) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] +(3) CometProject Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(5) Scan parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(6) CometProject Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] -(9) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [hd_demo_sk#5] -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_ship_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight -(11) Project [codegen id : 4] -Output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] +(9) CometProject Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] +Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(12) Scan parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [2]: [t_time_sk#7, t_hour#8] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [2]: [t_time_sk#7, t_hour#8] Condition : (((isnotnull(t_hour#8) AND (t_hour#8 >= 8)) AND (t_hour#8 <= 9)) AND isnotnull(t_time_sk#7)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] +(12) CometProject Input [2]: [t_time_sk#7, t_hour#8] +Arguments: [t_time_sk#7], [t_time_sk#7] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [t_time_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [t_time_sk#7] -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_time_sk#1] -Right keys [1]: [t_time_sk#7] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] +Right output [1]: [t_time_sk#7] +Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight -(18) Project [codegen id : 4] -Output [1]: [ws_web_page_sk#3] +(15) CometProject Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] +Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(19) Scan parquet spark_catalog.default.web_page +(16) CometNativeScan parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [wp_web_page_sk#9, wp_char_count#10] - -(21) Filter [codegen id : 3] +(17) CometFilter Input [2]: [wp_web_page_sk#9, wp_char_count#10] Condition : (((isnotnull(wp_char_count#10) AND (wp_char_count#10 >= 5000)) AND (wp_char_count#10 <= 5200)) AND isnotnull(wp_web_page_sk#9)) -(22) Project [codegen id : 3] -Output [1]: [wp_web_page_sk#9] +(18) CometProject Input [2]: [wp_web_page_sk#9, wp_char_count#10] +Arguments: [wp_web_page_sk#9], [wp_web_page_sk#9] -(23) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [wp_web_page_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [wp_web_page_sk#9] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_web_page_sk#3] -Right keys [1]: [wp_web_page_sk#9] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [1]: [ws_web_page_sk#3] +Right output [1]: [wp_web_page_sk#9] +Arguments: [ws_web_page_sk#3], [wp_web_page_sk#9], Inner, BuildRight -(25) Project [codegen id : 4] -Output: [] +(21) CometProject Input [2]: [ws_web_page_sk#3, wp_web_page_sk#9] -(26) HashAggregate [codegen id : 4] +(22) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#11] -Results [1]: [count#12] -(27) Exchange -Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [1]: [count#11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 10] -Input [1]: [count#12] +(24) CometHashAggregate +Input [1]: [count#11] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#13] -Results [1]: [count(1)#13 AS amc#14] -(29) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] +(25) CometColumnarToRow [codegen id : 2] +Input [1]: [amc#12] + +(26) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] +(27) CometFilter +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Condition : ((isnotnull(ws_ship_hdemo_sk#14) AND isnotnull(ws_sold_time_sk#13)) AND isnotnull(ws_web_page_sk#15)) -(31) Filter [codegen id : 8] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] -Condition : ((isnotnull(ws_ship_hdemo_sk#16) AND isnotnull(ws_sold_time_sk#15)) AND isnotnull(ws_web_page_sk#17)) +(28) CometProject +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] -(32) Project [codegen id : 8] -Output [3]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] +(29) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#17] -(33) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#19] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_ship_hdemo_sk#16] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [3]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] +Right output [1]: [hd_demo_sk#17] +Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight -(35) Project [codegen id : 8] -Output [2]: [ws_sold_time_sk#15, ws_web_page_sk#17] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, hd_demo_sk#19] +(31) CometProject +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] +Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(36) Scan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#20, t_hour#21] +(32) CometNativeScan parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [2]: [t_time_sk#20, t_hour#21] +(33) CometFilter +Input [2]: [t_time_sk#18, t_hour#19] +Condition : (((isnotnull(t_hour#19) AND (t_hour#19 >= 19)) AND (t_hour#19 <= 20)) AND isnotnull(t_time_sk#18)) -(38) Filter [codegen id : 6] -Input [2]: [t_time_sk#20, t_hour#21] -Condition : (((isnotnull(t_hour#21) AND (t_hour#21 >= 19)) AND (t_hour#21 <= 20)) AND isnotnull(t_time_sk#20)) +(34) CometProject +Input [2]: [t_time_sk#18, t_hour#19] +Arguments: [t_time_sk#18], [t_time_sk#18] -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#20] -Input [2]: [t_time_sk#20, t_hour#21] +(35) CometBroadcastExchange +Input [1]: [t_time_sk#18] +Arguments: [t_time_sk#18] -(40) BroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(36) CometBroadcastHashJoin +Left output [2]: [ws_sold_time_sk#13, ws_web_page_sk#15] +Right output [1]: [t_time_sk#18] +Arguments: [ws_sold_time_sk#13], [t_time_sk#18], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_sold_time_sk#15] -Right keys [1]: [t_time_sk#20] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 8] -Output [1]: [ws_web_page_sk#17] -Input [3]: [ws_sold_time_sk#15, ws_web_page_sk#17, t_time_sk#20] +(37) CometProject +Input [3]: [ws_sold_time_sk#13, ws_web_page_sk#15, t_time_sk#18] +Arguments: [ws_web_page_sk#15], [ws_web_page_sk#15] -(43) ReusedExchange [Reuses operator id: 23] -Output [1]: [wp_web_page_sk#22] +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [wp_web_page_sk#20] -(44) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_web_page_sk#17] -Right keys [1]: [wp_web_page_sk#22] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [1]: [ws_web_page_sk#15] +Right output [1]: [wp_web_page_sk#20] +Arguments: [ws_web_page_sk#15], [wp_web_page_sk#20], Inner, BuildRight -(45) Project [codegen id : 8] -Output: [] -Input [2]: [ws_web_page_sk#17, wp_web_page_sk#22] +(40) CometProject +Input [2]: [ws_web_page_sk#15, wp_web_page_sk#20] -(46) HashAggregate [codegen id : 8] +(41) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] -(47) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(42) CometExchange +Input [1]: [count#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(48) HashAggregate [codegen id : 9] -Input [1]: [count#24] +(43) CometHashAggregate +Input [1]: [count#21] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS pmc#26] -(49) BroadcastExchange -Input [1]: [pmc#26] -Arguments: IdentityBroadcastMode, [plan_id=7] +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [pmc#22] + +(45) BroadcastExchange +Input [1]: [pmc#22] +Arguments: IdentityBroadcastMode, [plan_id=3] -(50) BroadcastNestedLoopJoin [codegen id : 10] +(46) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(51) Project [codegen id : 10] -Output [1]: [(cast(amc#14 as decimal(15,4)) / cast(pmc#26 as decimal(15,4))) AS am_pm_ratio#27] -Input [2]: [amc#14, pmc#26] +(47) Project [codegen id : 2] +Output [1]: [(cast(amc#12 as decimal(15,4)) / cast(pmc#22 as decimal(15,4))) AS am_pm_ratio#23] +Input [2]: [amc#12, pmc#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/extended.txt new file mode 100644 index 0000000000..aa4016076b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/extended.txt @@ -0,0 +1,55 @@ +Project ++- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.time_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- BroadcastExchange + +- CometColumnarToRow + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 48 out of 51 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/simplified.txt index 9cf499b546..a13072cd91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/simplified.txt @@ -1,74 +1,52 @@ -WholeStageCodegen (10) +WholeStageCodegen (2) Project [amc,pmc] BroadcastNestedLoopJoin - HashAggregate [count] [count(1),amc,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_web_page_sk] - BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [wp_web_page_sk] - Filter [wp_char_count,wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometHashAggregate [count] [amc,count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] + CometProject [ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] + CometProject [ws_sold_time_sk,ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometBroadcastExchange [wp_web_page_sk] #4 + CometProject [wp_web_page_sk] + CometFilter [wp_web_page_sk,wp_char_count] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 - WholeStageCodegen (9) - HashAggregate [count] [count(1),pmc,count] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #6 - WholeStageCodegen (8) - HashAggregate [count,count] - Project - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_web_page_sk] - BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] - InputAdapter - ReusedExchange [wp_web_page_sk] #4 + CometHashAggregate [count] [pmc,count(1)] + CometExchange #6 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] + CometProject [ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] + CometProject [ws_sold_time_sk,ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #7 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/explain.txt index 1083b68c93..42aba65d76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/explain.txt @@ -1,271 +1,247 @@ == Physical Plan == -* Sort (47) -+- Exchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (22) - : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : :- * Project (16) - : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : :- * Project (9) - : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.call_center (1) - : : : : : +- BroadcastExchange (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (4) - : : : : +- BroadcastExchange (14) - : : : : +- * Project (13) - : : : : +- * Filter (12) - : : : : +- * ColumnarToRow (11) - : : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : : +- BroadcastExchange (20) - : : : +- * Filter (19) - : : : +- * ColumnarToRow (18) - : : : +- Scan parquet spark_catalog.default.customer (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * Filter (25) - : : +- * ColumnarToRow (24) - : : +- Scan parquet spark_catalog.default.customer_address (23) - : +- BroadcastExchange (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.customer_demographics (30) - +- BroadcastExchange (40) - +- * Project (39) - +- * Filter (38) - +- * ColumnarToRow (37) - +- Scan parquet spark_catalog.default.household_demographics (36) - - -(1) Scan parquet spark_catalog.default.call_center +* CometColumnarToRow (43) ++- CometSort (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometProject (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometFilter (5) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (4) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometNativeScan parquet spark_catalog.default.customer (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometNativeScan parquet spark_catalog.default.customer_demographics (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.household_demographics (32) + + +(1) CometNativeScan parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Condition : isnotnull(cc_call_center_sk#1) -(3) Filter [codegen id : 7] +(3) CometProject Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Condition : isnotnull(cc_call_center_sk#1) +Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cc_call_center_id#2, 16, true, false, true) AS cc_call_center_id#5, cc_name#3, cc_manager#4] -(4) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] +(4) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#8)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] +(5) CometFilter +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_call_center_sk#7) AND isnotnull(cr_returning_customer_sk#6)) -(6) Filter [codegen id : 1] -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) +(6) CometBroadcastExchange +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] -(7) BroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4] +Right output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cc_call_center_sk#1] -Right keys [1]: [cr_call_center_sk#6] -Join type: Inner -Join condition: None +(8) CometProject +Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(9) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +(11) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] -(14) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(15) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cr_returned_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight -(16) Project [codegen id : 7] -Output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] +(14) CometProject +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(17) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] +(15) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] +(16) CometFilter +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) -(19) Filter [codegen id : 3] -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) +(17) CometBroadcastExchange +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] -(20) BroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cr_returning_customer_sk#5] -Right keys [1]: [c_customer_sk#12] -Join type: Inner -Join condition: None +(19) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] -(22) Project [codegen id : 7] -Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(23) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] +(20) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] - -(25) Filter [codegen id : 4] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) +(21) CometFilter +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) -(26) Project [codegen id : 4] -Output [1]: [ca_address_sk#16] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] +(22) CometProject +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(27) BroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#15] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight -(29) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] +(25) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] -(30) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(26) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree ))), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(27) CometFilter +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) -(32) Filter [codegen id : 5] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) +(28) CometProject +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] -(33) BroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#13] -Right keys [1]: [cd_demo_sk#18] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight -(35) Project [codegen id : 7] -Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(31) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] -(36) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] +(32) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), StringStartsWith(hd_buy_potential,Unknown), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] - -(38) Filter [codegen id : 6] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) - -(39) Project [codegen id : 6] -Output [1]: [hd_demo_sk#21] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] - -(40) BroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_hdemo_sk#14] -Right keys [1]: [hd_demo_sk#21] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] - -(43) HashAggregate [codegen id : 7] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum#23] -Results [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] - -(44) Exchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 8] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#7))#25] -Results [4]: [cc_call_center_id#2 AS Call_Center#26, cc_name#3 AS Call_Center_Name#27, cc_manager#4 AS Manager#28, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#25,17,2) AS Returns_Loss#29] - -(46) Exchange -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: rangepartitioning(Returns_Loss#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(47) Sort [codegen id : 9] -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: [Returns_Loss#29 DESC NULLS LAST], true, 0 +(33) CometFilter +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#25, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#24)) + +(34) CometProject +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] + +(35) CometBroadcastExchange +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] + +(36) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight + +(37) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] + +(38) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] + +(39) CometExchange +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] + +(41) CometExchange +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: rangepartitioning(Returns_Loss#30 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometSort +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30], [Returns_Loss#30 DESC NULLS LAST] + +(43) CometColumnarToRow [codegen id : 1] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/extended.txt new file mode 100644 index 0000000000..f3d3dc8b3b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/extended.txt @@ -0,0 +1,45 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.call_center + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.household_demographics + +Comet accelerated 42 out of 42 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/simplified.txt index ac1b59c58b..36fc2b1b5b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/simplified.txt @@ -1,71 +1,45 @@ -WholeStageCodegen (9) - Sort [Returns_Loss] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [Returns_Loss] #1 - WholeStageCodegen (8) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum] - InputAdapter - Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - WholeStageCodegen (7) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - BroadcastHashJoin [cr_returning_customer_sk,c_customer_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] - BroadcastHashJoin [cc_call_center_sk,cr_call_center_sk] - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cr_call_center_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] + CometExchange [Returns_Loss] #1 + CometHashAggregate [cd_marital_status,cd_education_status,sum] [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,sum(UnscaledValue(cr_net_loss))] + CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 + CometHashAggregate [cr_net_loss] [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 + CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/explain.txt index 3175a02337..6f3e34a6dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/explain.txt @@ -1,197 +1,173 @@ == Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- * Project (20) - : +- * BroadcastHashJoin Inner BuildRight (19) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.web_sales (11) - : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.date_dim (14) - +- ReusedExchange (28) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (30) ++- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometNativeScan parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#4] +(5) CometProject Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [i_item_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#4] -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Right output [1]: [i_item_sk#4] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(10) Project [codegen id : 6] -Output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +(8) CometProject Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -(11) Scan parquet spark_catalog.default.web_sales +(9) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#8)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] - -(13) Filter [codegen id : 3] +(10) CometFilter Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Condition : isnotnull(ws_item_sk#6) -(14) Scan parquet spark_catalog.default.date_dim +(11) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(16) Filter [codegen id : 2] +(12) CometFilter Input [2]: [d_date_sk#9, d_date#10] Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) -(17) Project [codegen id : 2] -Output [1]: [d_date_sk#9] +(13) CometProject Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(18) BroadcastExchange +(14) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#9] -(19) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(20) Project [codegen id : 3] -Output [2]: [ws_item_sk#6, ws_ext_discount_amt#7] +(16) CometProject Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] +Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] -(21) HashAggregate [codegen id : 3] +(17) CometHashAggregate Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] Keys [1]: [ws_item_sk#6] Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [ws_item_sk#6, sum#13, count#14] -(22) Exchange -Input [3]: [ws_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(18) CometExchange +Input [3]: [ws_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 4] -Input [3]: [ws_item_sk#6, sum#13, count#14] +(19) CometHashAggregate +Input [3]: [ws_item_sk#6, sum#11, count#12] Keys [1]: [ws_item_sk#6] Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(ws_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -(24) Filter [codegen id : 4] -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#16) +(20) CometFilter +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) -(25) BroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [ws_item_sk#6] -Join type: Inner -Join condition: (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#16) +(22) CometBroadcastHashJoin +Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight -(27) Project [codegen id : 6] -Output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] +(23) CometProject +Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] -(28) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#17] +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#9] -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(30) Project [codegen id : 6] -Output [1]: [ws_ext_discount_amt#2] -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#17] +(26) CometProject +Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#9] +Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] -(31) HashAggregate [codegen id : 6] +(27) CometHashAggregate Input [1]: [ws_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#18] -Results [1]: [sum#19] -(32) Exchange -Input [1]: [sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [1]: [sum#14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(33) HashAggregate [codegen id : 7] -Input [1]: [sum#19] +(29) CometHashAggregate +Input [1]: [sum#14] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#20,17,2) AS Excess Discount Amount #21] + +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [Excess Discount Amount #15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/extended.txt new file mode 100644 index 0000000000..f9fb54b694 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/simplified.txt index 6f9df5402f..217c245298 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/simplified.txt @@ -1,49 +1,32 @@ -WholeStageCodegen (7) - HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [ws_ext_discount_amt] [sum,sum] - Project [ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_discount_amt,ws_sold_date_sk] - BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] - Project [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk,ws_ext_discount_amt] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Filter [(1.3 * avg(ws_ext_discount_amt))] - HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] - Project [ws_item_sk,ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - ReusedExchange [d_date_sk] #5 + CometHashAggregate [sum] [Excess Discount Amount ,sum(UnscaledValue(ws_ext_discount_amt))] + CometExchange #1 + CometHashAggregate [ws_ext_discount_amt] [sum] + CometProject [ws_ext_discount_amt] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk,(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 + CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] + CometExchange [ws_item_sk] #4 + CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] + CometProject [ws_item_sk,ws_ext_discount_amt] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/explain.txt index 172c035654..3f38adcd39 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/explain.txt @@ -1,138 +1,124 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * Project (20) - +- * BroadcastHashJoin Inner BuildRight (19) - :- * Project (13) - : +- * SortMergeJoin Inner (12) - : :- * Sort (5) - : : +- Exchange (4) - : : +- * Project (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- * Sort (11) - : +- Exchange (10) - : +- * Project (9) - : +- * Filter (8) - : +- * ColumnarToRow (7) - : +- Scan parquet spark_catalog.default.store_returns (6) - +- BroadcastExchange (18) - +- * Project (17) - +- * Filter (16) - +- * ColumnarToRow (15) - +- Scan parquet spark_catalog.default.reason (14) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (22) ++- CometTakeOrderedAndProject (21) + +- CometHashAggregate (20) + +- CometExchange (19) + +- CometHashAggregate (18) + +- CometProject (17) + +- CometBroadcastHashJoin (16) + :- CometProject (11) + : +- CometSortMergeJoin (10) + : :- CometSort (4) + : : +- CometExchange (3) + : : +- CometProject (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometSort (9) + : +- CometExchange (8) + : +- CometProject (7) + : +- CometFilter (6) + : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + +- CometBroadcastExchange (15) + +- CometProject (14) + +- CometFilter (13) + +- CometNativeScan parquet spark_catalog.default.reason (12) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -(3) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] - -(4) Exchange +(3) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_item_sk#7) AND isnotnull(sr_ticket_number#9)) AND isnotnull(sr_reason_sk#8)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +(7) CometProject Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -(10) Exchange +(8) CometExchange Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#3] -Right keys [2]: [sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(10) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] +Right output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9], Inner -(13) Project [codegen id : 6] -Output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] +(11) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(14) Scan parquet spark_catalog.default.reason +(12) CometNativeScan parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] -PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28 ), IsNotNull(r_reason_sk)] +PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] +(13) CometFilter Input [2]: [r_reason_sk#12, r_reason_desc#13] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#13, 100, true, false, true) = reason 28 ) AND isnotnull(r_reason_sk#12)) -(16) Filter [codegen id : 5] +(14) CometProject Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) +Arguments: [r_reason_sk#12], [r_reason_sk#12] -(17) Project [codegen id : 5] -Output [1]: [r_reason_sk#12] -Input [2]: [r_reason_sk#12, r_reason_desc#13] - -(18) BroadcastExchange +(15) CometBroadcastExchange Input [1]: [r_reason_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [r_reason_sk#12] -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_reason_sk#8] -Right keys [1]: [r_reason_sk#12] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] +Right output [1]: [r_reason_sk#12] +Arguments: [sr_reason_sk#8], [r_reason_sk#12], Inner, BuildRight -(20) Project [codegen id : 6] -Output [2]: [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] +(17) CometProject Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10, r_reason_sk#12] +Arguments: [ss_customer_sk#2, act_sales#14], [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [2]: [ss_customer_sk#2, act_sales#14] Keys [1]: [ss_customer_sk#2] Functions [1]: [partial_sum(act_sales#14)] -Aggregate Attributes [2]: [sum#15, isEmpty#16] -Results [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -(22) Exchange -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) HashAggregate [codegen id : 7] -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] +(20) CometHashAggregate +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] Keys [1]: [ss_customer_sk#2] Functions [1]: [sum(act_sales#14)] -Aggregate Attributes [1]: [sum(act_sales#14)#19] -Results [2]: [ss_customer_sk#2, sum(act_sales#14)#19 AS sumsales#20] -(24) TakeOrderedAndProject -Input [2]: [ss_customer_sk#2, sumsales#20] -Arguments: 100, [sumsales#20 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#20] +(21) CometTakeOrderedAndProject +Input [2]: [ss_customer_sk#2, sumsales#17] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, 0, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] + +(22) CometColumnarToRow [codegen id : 1] +Input [2]: [ss_customer_sk#2, sumsales#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/extended.txt new file mode 100644 index 0000000000..7203a678b0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/extended.txt @@ -0,0 +1,24 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.reason + +Comet accelerated 21 out of 21 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/simplified.txt index 350956593a..c0b74010df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/simplified.txt @@ -1,40 +1,24 @@ -TakeOrderedAndProject [sumsales,ss_customer_sk] - WholeStageCodegen (7) - HashAggregate [ss_customer_sk,sum,isEmpty] [sum(act_sales),sumsales,sum,isEmpty] - InputAdapter - Exchange [ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [ss_customer_sk,act_sales] [sum,isEmpty,sum,isEmpty] - Project [ss_customer_sk,sr_return_quantity,ss_quantity,ss_sales_price] - BroadcastHashJoin [sr_reason_sk,r_reason_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #2 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - Filter [sr_item_sk,sr_ticket_number,sr_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Project [r_reason_sk] - Filter [r_reason_desc,r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ss_customer_sk,sumsales] + CometHashAggregate [sum,isEmpty] [ss_customer_sk,sumsales,sum(act_sales)] + CometExchange [ss_customer_sk] #1 + CometHashAggregate [act_sales] [ss_customer_sk,sum,isEmpty] + CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometExchange [ss_item_sk,ss_ticket_number] #2 + CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometExchange [sr_item_sk,sr_ticket_number] #3 + CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [r_reason_sk] #4 + CometProject [r_reason_sk] + CometFilter [r_reason_sk,r_reason_desc] + CometNativeScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/explain.txt index 91592f54ab..1caeb8ca76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/explain.txt @@ -1,260 +1,235 @@ == Physical Plan == -* HashAggregate (45) -+- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- * Sort (11) - : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.web_sales (7) - : : : +- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.web_returns (14) - : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) - : : +- Scan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.web_site (34) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometNativeScan parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometNativeScan parquet spark_catalog.default.web_site (29) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) -(4) Project [codegen id : 1] -Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(3) CometProject Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(5) Exchange +(4) CometExchange Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: [ws_order_number#5 ASC NULLS FIRST], false, 0 +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.web_sales +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] - -(9) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#9, ws_order_number#10] +(7) CometProject Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] -(10) Exchange +(8) CometExchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: [ws_order_number#10 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_order_number#10 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 5] -Left keys [1]: [ws_order_number#5] -Right keys [1]: [ws_order_number#10] -Join type: LeftSemi -Join condition: NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) +(10) CometSortMergeJoin +Left output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [2]: [ws_warehouse_sk#9, ws_order_number#10] +Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) -(13) Project [codegen id : 5] -Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(11) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(14) Scan parquet spark_catalog.default.web_returns +(12) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] ReadSchema: struct -(15) ColumnarToRow [codegen id : 6] +(13) CometProject Input [2]: [wr_order_number#12, wr_returned_date_sk#13] +Arguments: [wr_order_number#12], [wr_order_number#12] -(16) Project [codegen id : 6] -Output [1]: [wr_order_number#12] -Input [2]: [wr_order_number#12, wr_returned_date_sk#13] - -(17) Exchange +(14) CometExchange Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(18) Sort [codegen id : 7] +(15) CometSort Input [1]: [wr_order_number#12] -Arguments: [wr_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [wr_order_number#12], [wr_order_number#12 ASC NULLS FIRST] -(19) SortMergeJoin [codegen id : 11] -Left keys [1]: [ws_order_number#5] -Right keys [1]: [wr_order_number#12] -Join type: LeftAnti -Join condition: None +(16) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [wr_order_number#12] +Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(20) Scan parquet spark_catalog.default.date_dim +(17) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] +(18) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-01)) AND (d_date#15 <= 1999-04-02)) AND isnotnull(d_date_sk#14)) -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] +(19) CometProject Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] -(24) BroadcastExchange +(20) CometBroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#14] -(25) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [d_date_sk#14] +Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight -(26) Project [codegen id : 11] -Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(22) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] +Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(27) Scan parquet spark_catalog.default.customer_address +(23) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] +(24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#16)) -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] +(25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] -(31) BroadcastExchange +(26) CometBroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#16] -(32) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [ca_address_sk#16] +Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight -(33) Project [codegen id : 11] -Output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(28) CometProject Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] +Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(34) Scan parquet spark_catalog.default.web_site +(29) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(30) CometFilter Input [2]: [web_site_sk#18, web_company_name#19] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#19, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#18)) -(36) Filter [codegen id : 10] +(31) CometProject Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) +Arguments: [web_site_sk#18], [web_site_sk#18] -(37) Project [codegen id : 10] -Output [1]: [web_site_sk#18] -Input [2]: [web_site_sk#18, web_company_name#19] - -(38) BroadcastExchange +(32) CometBroadcastExchange Input [1]: [web_site_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [web_site_sk#18] -(39) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#18] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [web_site_sk#18] +Arguments: [ws_web_site_sk#3], [web_site_sk#18], Inner, BuildRight -(40) Project [codegen id : 11] -Output [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(34) CometProject Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#18] +Arguments: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(41) HashAggregate [codegen id : 11] +(35) CometHashAggregate Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Keys [1]: [ws_order_number#5] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] -(42) HashAggregate [codegen id : 11] -Input [3]: [ws_order_number#5, sum#22, sum#23] +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] + +(37) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] Keys [1]: [ws_order_number#5] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23] +Results [3]: [ws_order_number#5, sum#20, sum#21] -(43) HashAggregate [codegen id : 11] -Input [3]: [ws_order_number#5, sum#22, sum#23] +(38) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] -(44) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(45) HashAggregate [codegen id : 12] -Input [3]: [sum#22, sum#23, count#25] +(40) CometHashAggregate +Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#21,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/extended.txt new file mode 100644 index 0000000000..e5c7b8688a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/extended.txt @@ -0,0 +1,43 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 37 out of 39 eligible operators (94%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/simplified.txt index 86981cc808..feab73bcd6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/simplified.txt @@ -1,74 +1,45 @@ -WholeStageCodegen (12) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] +WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (5) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #2 - WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #3 - WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [wr_order_number] - InputAdapter - Exchange [wr_order_number] #4 - WholeStageCodegen (6) - Project [wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/explain.txt index 194134249e..e49aacd1b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/explain.txt @@ -1,330 +1,302 @@ == Physical Plan == -* HashAggregate (58) -+- Exchange (57) - +- * HashAggregate (56) - +- * HashAggregate (55) - +- * HashAggregate (54) - +- * Project (53) - +- * BroadcastHashJoin Inner BuildRight (52) - :- * Project (46) - : +- * BroadcastHashJoin Inner BuildRight (45) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * SortMergeJoin LeftSemi (32) - : : : :- * SortMergeJoin LeftSemi (17) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (16) - : : : : +- * SortMergeJoin Inner (15) - : : : : :- * Sort (12) - : : : : : +- Exchange (11) - : : : : : +- * Project (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) - : : : : : +- Scan parquet spark_catalog.default.web_sales (7) - : : : : +- * Sort (14) - : : : : +- ReusedExchange (13) - : : : +- * Project (31) - : : : +- * SortMergeJoin Inner (30) - : : : :- * Sort (23) - : : : : +- Exchange (22) - : : : : +- * Project (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.web_returns (18) - : : : +- * Project (29) - : : : +- * SortMergeJoin Inner (28) - : : : :- * Sort (25) - : : : : +- ReusedExchange (24) - : : : +- * Sort (27) - : : : +- ReusedExchange (26) - : : +- BroadcastExchange (37) - : : +- * Project (36) - : : +- * Filter (35) - : : +- * ColumnarToRow (34) - : : +- Scan parquet spark_catalog.default.date_dim (33) - : +- BroadcastExchange (44) - : +- * Project (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer_address (40) - +- BroadcastExchange (51) - +- * Project (50) - +- * Filter (49) - +- * ColumnarToRow (48) - +- Scan parquet spark_catalog.default.web_site (47) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (54) ++- CometHashAggregate (53) + +- CometColumnarExchange (52) + +- * HashAggregate (51) + +- * HashAggregate (50) + +- * CometColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometBroadcastHashJoin (46) + :- CometProject (41) + : +- CometBroadcastHashJoin (40) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometSortMergeJoin (29) + : : : :- CometSortMergeJoin (15) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometProject (14) + : : : : +- CometSortMergeJoin (13) + : : : : :- CometSort (10) + : : : : : +- CometExchange (9) + : : : : : +- CometProject (8) + : : : : : +- CometFilter (7) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (6) + : : : : +- CometSort (12) + : : : : +- ReusedExchange (11) + : : : +- CometProject (28) + : : : +- CometSortMergeJoin (27) + : : : :- CometSort (20) + : : : : +- CometExchange (19) + : : : : +- CometProject (18) + : : : : +- CometFilter (17) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (16) + : : : +- CometProject (26) + : : : +- CometSortMergeJoin (25) + : : : :- CometSort (22) + : : : : +- ReusedExchange (21) + : : : +- CometSort (24) + : : : +- ReusedExchange (23) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : +- CometBroadcastExchange (39) + : +- CometProject (38) + : +- CometFilter (37) + : +- CometNativeScan parquet spark_catalog.default.customer_address (36) + +- CometBroadcastExchange (45) + +- CometProject (44) + +- CometFilter (43) + +- CometNativeScan parquet spark_catalog.default.web_site (42) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) -(4) Project [codegen id : 1] -Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(3) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(5) Exchange +(4) CometExchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: [ws_order_number#4 ASC NULLS FIRST], false, 0 +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.web_sales +(6) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) -(10) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#8, ws_order_number#9] +(8) CometProject Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] -(11) Exchange +(9) CometExchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_order_number#9 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] -(13) ReusedExchange [Reuses operator id: 11] +(11) ReusedExchange [Reuses operator id: 9] Output [2]: [ws_warehouse_sk#11, ws_order_number#12] -(14) Sort [codegen id : 6] +(12) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(15) SortMergeJoin [codegen id : 7] -Left keys [1]: [ws_order_number#9] -Right keys [1]: [ws_order_number#12] -Join type: Inner -Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) +(13) CometSortMergeJoin +Left output [2]: [ws_warehouse_sk#8, ws_order_number#9] +Right output [2]: [ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#12], Inner, NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(16) Project [codegen id : 7] -Output [1]: [ws_order_number#9] +(14) CometProject Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#9] -(17) SortMergeJoin [codegen id : 8] -Left keys [1]: [ws_order_number#4] -Right keys [1]: [ws_order_number#9] -Join type: LeftSemi -Join condition: None +(15) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [ws_order_number#9] +Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(18) Scan parquet spark_catalog.default.web_returns +(16) CometNativeScan parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 9] -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] - -(20) Filter [codegen id : 9] +(17) CometFilter Input [2]: [wr_order_number#13, wr_returned_date_sk#14] Condition : isnotnull(wr_order_number#13) -(21) Project [codegen id : 9] -Output [1]: [wr_order_number#13] +(18) CometProject Input [2]: [wr_order_number#13, wr_returned_date_sk#14] +Arguments: [wr_order_number#13], [wr_order_number#13] -(22) Exchange +(19) CometExchange Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) Sort [codegen id : 10] +(20) CometSort Input [1]: [wr_order_number#13] -Arguments: [wr_order_number#13 ASC NULLS FIRST], false, 0 +Arguments: [wr_order_number#13], [wr_order_number#13 ASC NULLS FIRST] -(24) ReusedExchange [Reuses operator id: 11] +(21) ReusedExchange [Reuses operator id: 9] Output [2]: [ws_warehouse_sk#8, ws_order_number#9] -(25) Sort [codegen id : 12] +(22) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_order_number#9 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] -(26) ReusedExchange [Reuses operator id: 11] +(23) ReusedExchange [Reuses operator id: 9] Output [2]: [ws_warehouse_sk#11, ws_order_number#12] -(27) Sort [codegen id : 14] +(24) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(28) SortMergeJoin [codegen id : 15] -Left keys [1]: [ws_order_number#9] -Right keys [1]: [ws_order_number#12] -Join type: Inner -Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) +(25) CometSortMergeJoin +Left output [2]: [ws_warehouse_sk#8, ws_order_number#9] +Right output [2]: [ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#12], Inner, NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(29) Project [codegen id : 15] -Output [1]: [ws_order_number#9] +(26) CometProject Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#9] -(30) SortMergeJoin [codegen id : 16] -Left keys [1]: [wr_order_number#13] -Right keys [1]: [ws_order_number#9] -Join type: Inner -Join condition: None +(27) CometSortMergeJoin +Left output [1]: [wr_order_number#13] +Right output [1]: [ws_order_number#9] +Arguments: [wr_order_number#13], [ws_order_number#9], Inner -(31) Project [codegen id : 16] -Output [1]: [wr_order_number#13] +(28) CometProject Input [2]: [wr_order_number#13, ws_order_number#9] +Arguments: [wr_order_number#13], [wr_order_number#13] -(32) SortMergeJoin [codegen id : 20] -Left keys [1]: [ws_order_number#4] -Right keys [1]: [wr_order_number#13] -Join type: LeftSemi -Join condition: None +(29) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [wr_order_number#13] +Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(33) Scan parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#15, d_date#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 17] -Input [2]: [d_date_sk#15, d_date#16] - -(35) Filter [codegen id : 17] +(31) CometFilter Input [2]: [d_date_sk#15, d_date#16] Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 1999-02-01)) AND (d_date#16 <= 1999-04-02)) AND isnotnull(d_date_sk#15)) -(36) Project [codegen id : 17] -Output [1]: [d_date_sk#15] +(32) CometProject Input [2]: [d_date_sk#15, d_date#16] +Arguments: [d_date_sk#15], [d_date_sk#15] -(37) BroadcastExchange +(33) CometBroadcastExchange Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#15] -(38) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [d_date_sk#15] +Arguments: [ws_ship_date_sk#1], [d_date_sk#15], Inner, BuildRight -(39) Project [codegen id : 20] -Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(35) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#15] +Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(40) Scan parquet spark_catalog.default.customer_address +(36) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 18] -Input [2]: [ca_address_sk#17, ca_state#18] - -(42) Filter [codegen id : 18] +(37) CometFilter Input [2]: [ca_address_sk#17, ca_state#18] -Condition : ((isnotnull(ca_state#18) AND (ca_state#18 = IL)) AND isnotnull(ca_address_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#17)) -(43) Project [codegen id : 18] -Output [1]: [ca_address_sk#17] +(38) CometProject Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(44) BroadcastExchange +(39) CometBroadcastExchange Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#17] -(45) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(40) CometBroadcastHashJoin +Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [ca_address_sk#17] +Arguments: [ws_ship_addr_sk#2], [ca_address_sk#17], Inner, BuildRight -(46) Project [codegen id : 20] -Output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(41) CometProject Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#17] +Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(47) Scan parquet spark_catalog.default.web_site +(42) CometNativeScan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#19, web_company_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 19] +(43) CometFilter Input [2]: [web_site_sk#19, web_company_name#20] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#20, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#19)) -(49) Filter [codegen id : 19] +(44) CometProject Input [2]: [web_site_sk#19, web_company_name#20] -Condition : ((isnotnull(web_company_name#20) AND (web_company_name#20 = pri )) AND isnotnull(web_site_sk#19)) +Arguments: [web_site_sk#19], [web_site_sk#19] -(50) Project [codegen id : 19] -Output [1]: [web_site_sk#19] -Input [2]: [web_site_sk#19, web_company_name#20] - -(51) BroadcastExchange +(45) CometBroadcastExchange Input [1]: [web_site_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [web_site_sk#19] -(52) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#19] -Join type: Inner -Join condition: None +(46) CometBroadcastHashJoin +Left output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [web_site_sk#19] +Arguments: [ws_web_site_sk#3], [web_site_sk#19], Inner, BuildRight -(53) Project [codegen id : 20] -Output [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(47) CometProject Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#19] +Arguments: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(54) HashAggregate [codegen id : 20] +(48) CometHashAggregate Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Keys [1]: [ws_order_number#4] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] -(55) HashAggregate [codegen id : 20] -Input [3]: [ws_order_number#4, sum#23, sum#24] +(49) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#4, sum#21, sum#22] + +(50) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#4, sum#21, sum#22] Keys [1]: [ws_order_number#4] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#23, sum(UnscaledValue(ws_net_profit#6))#24] +Results [3]: [ws_order_number#4, sum#21, sum#22] -(56) HashAggregate [codegen id : 20] -Input [3]: [ws_order_number#4, sum#23, sum#24] +(51) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#4, sum#21, sum#22] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [sum#23, sum#24, count#26] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#23, sum(UnscaledValue(ws_net_profit#6))#24, count(ws_order_number#4)#25] +Results [3]: [sum#21, sum#22, count#26] -(57) Exchange -Input [3]: [sum#23, sum#24, count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(52) CometColumnarExchange +Input [3]: [sum#21, sum#22, count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(58) HashAggregate [codegen id : 21] -Input [3]: [sum#23, sum#24, count#26] +(53) CometHashAggregate +Input [3]: [sum#21, sum#22, count#26] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [count(ws_order_number#4)#25 AS order count #27, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#21,17,2) AS total shipping cost #28, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#22,17,2) AS total net profit #29] + +(54) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #27, total shipping cost #28, total net profit #29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/extended.txt new file mode 100644 index 0000000000..cbba03d28d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/extended.txt @@ -0,0 +1,65 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: Unsupported aggregation mode PartialMerge] + +- CometColumnarToRow + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometSortMergeJoin + : : : :- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 59 out of 61 eligible operators (96%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/simplified.txt index f628b29094..15f29c507c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/simplified.txt @@ -1,102 +1,58 @@ -WholeStageCodegen (21) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] +WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (20) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (8) - SortMergeJoin [ws_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (2) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #2 - WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (4) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #3 - WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - Filter [ws_order_number,ws_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (6) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (16) - Project [wr_order_number] - SortMergeJoin [wr_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (10) - Sort [wr_order_number] - InputAdapter - Exchange [wr_order_number] #4 - WholeStageCodegen (9) - Project [wr_order_number] - Filter [wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (15) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (12) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (14) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (17) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (18) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (19) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometProject [wr_order_number] + CometSortMergeJoin [wr_order_number,ws_order_number] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/explain.txt index 067597a654..db701aae34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/explain.txt @@ -1,163 +1,143 @@ == Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.household_demographics (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.time_dim (12) - +- BroadcastExchange (23) - +- * Project (22) - +- * Filter (21) - +- * ColumnarToRow (20) - +- Scan parquet spark_catalog.default.store (19) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (25) ++- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometNativeScan parquet spark_catalog.default.household_demographics (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.time_dim (10) + +- CometBroadcastExchange (19) + +- CometProject (18) + +- CometFilter (17) + +- CometNativeScan parquet spark_catalog.default.store (16) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +(3) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(5) Scan parquet spark_catalog.default.household_demographics +(4) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(6) CometProject Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] -(9) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [hd_demo_sk#5] -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight -(11) Project [codegen id : 4] -Output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +(9) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] +Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(12) Scan parquet spark_catalog.default.time_dim +(10) CometNativeScan parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] Condition : ((((isnotnull(t_hour#8) AND isnotnull(t_minute#9)) AND (t_hour#8 = 20)) AND (t_minute#9 >= 30)) AND isnotnull(t_time_sk#7)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] +(12) CometProject Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] +Arguments: [t_time_sk#7], [t_time_sk#7] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [t_time_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [t_time_sk#7] -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_time_sk#1] -Right keys [1]: [t_time_sk#7] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +Right output [1]: [t_time_sk#7] +Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight -(18) Project [codegen id : 4] -Output [1]: [ss_store_sk#3] +(15) CometProject Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] +Arguments: [ss_store_sk#3], [ss_store_sk#3] -(19) Scan parquet spark_catalog.default.store +(16) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#10, s_store_name#11] - -(21) Filter [codegen id : 3] +(17) CometFilter Input [2]: [s_store_sk#10, s_store_name#11] Condition : ((isnotnull(s_store_name#11) AND (s_store_name#11 = ese)) AND isnotnull(s_store_sk#10)) -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#10] +(18) CometProject Input [2]: [s_store_sk#10, s_store_name#11] +Arguments: [s_store_sk#10], [s_store_sk#10] -(23) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [s_store_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#10] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#3] +Right output [1]: [s_store_sk#10] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(25) Project [codegen id : 4] -Output: [] +(21) CometProject Input [2]: [ss_store_sk#3, s_store_sk#10] -(26) HashAggregate [codegen id : 4] +(22) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#12] -Results [1]: [count#13] -(27) Exchange -Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [1]: [count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [1]: [count#13] +(24) CometHashAggregate +Input [1]: [count#12] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#14] -Results [1]: [count(1)#14 AS count(1)#15] + +(25) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/extended.txt new file mode 100644 index 0000000000..170d3b7879 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.time_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 24 out of 24 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/simplified.txt index 9f6ea6e9b4..b40f41659f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/simplified.txt @@ -1,41 +1,27 @@ -WholeStageCodegen (5) - HashAggregate [count] [count(1),count(1),count] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometHashAggregate [count] [count(1),count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometNativeScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_store_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/explain.txt index 235e92c3ed..b5a88b6563 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/explain.txt @@ -1,167 +1,144 @@ == Physical Plan == -* HashAggregate (27) -+- Exchange (26) - +- * HashAggregate (25) - +- * Project (24) - +- * SortMergeJoin FullOuter (23) - :- * Sort (13) - : +- * HashAggregate (12) - : +- Exchange (11) - : +- * HashAggregate (10) - : +- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Project (6) - : +- * Filter (5) - : +- * ColumnarToRow (4) - : +- Scan parquet spark_catalog.default.date_dim (3) - +- * Sort (22) - +- * HashAggregate (21) - +- Exchange (20) - +- * HashAggregate (19) - +- * Project (18) - +- * BroadcastHashJoin Inner BuildRight (17) - :- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.catalog_sales (14) - +- ReusedExchange (16) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (25) ++- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometSortMergeJoin (20) + :- CometSort (11) + : +- CometHashAggregate (10) + : +- CometExchange (9) + : +- CometHashAggregate (8) + : +- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (5) + : +- CometProject (4) + : +- CometFilter (3) + : +- CometNativeScan parquet spark_catalog.default.date_dim (2) + +- CometSort (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometProject (15) + +- CometBroadcastHashJoin (14) + :- CometNativeScan parquet spark_catalog.default.catalog_sales (12) + +- ReusedExchange (13) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] - -(3) Scan parquet spark_catalog.default.date_dim +(2) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(4) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(5) Filter [codegen id : 1] +(3) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(6) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(4) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(8) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(9) Project [codegen id : 2] -Output [2]: [ss_item_sk#1, ss_customer_sk#2] +(7) CometProject Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] -(10) HashAggregate [codegen id : 2] +(8) CometHashAggregate Input [2]: [ss_item_sk#1, ss_customer_sk#2] Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] -Aggregate Attributes: [] -Results [2]: [ss_customer_sk#2, ss_item_sk#1] -(11) Exchange +(9) CometExchange Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) HashAggregate [codegen id : 3] +(10) CometHashAggregate Input [2]: [ss_customer_sk#2, ss_item_sk#1] Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] -Aggregate Attributes: [] -Results [2]: [ss_customer_sk#2 AS customer_sk#6, ss_item_sk#1 AS item_sk#7] -(13) Sort [codegen id : 3] +(11) CometSort Input [2]: [customer_sk#6, item_sk#7] -Arguments: [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] -(14) Scan parquet spark_catalog.default.catalog_sales +(12) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#10)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] - -(16) ReusedExchange [Reuses operator id: 7] +(13) ReusedExchange [Reuses operator id: 5] Output [1]: [d_date_sk#11] -(17) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight -(18) Project [codegen id : 5] -Output [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +(15) CometProject Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] +Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] -(19) HashAggregate [codegen id : 5] +(16) CometHashAggregate Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] -Aggregate Attributes: [] -Results [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -(20) Exchange +(17) CometExchange Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] -Aggregate Attributes: [] -Results [2]: [cs_bill_customer_sk#8 AS customer_sk#12, cs_item_sk#9 AS item_sk#13] -(22) Sort [codegen id : 6] +(19) CometSort Input [2]: [customer_sk#12, item_sk#13] -Arguments: [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST], false, 0 +Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] -(23) SortMergeJoin [codegen id : 7] -Left keys [2]: [customer_sk#6, item_sk#7] -Right keys [2]: [customer_sk#12, item_sk#13] -Join type: FullOuter -Join condition: None +(20) CometSortMergeJoin +Left output [2]: [customer_sk#6, item_sk#7] +Right output [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter -(24) Project [codegen id : 7] -Output [2]: [customer_sk#6, customer_sk#12] +(21) CometProject Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] -(25) HashAggregate [codegen id : 7] +(22) CometHashAggregate Input [2]: [customer_sk#6, customer_sk#12] Keys: [] Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum#14, sum#15, sum#16] -Results [3]: [sum#17, sum#18, sum#19] -(26) Exchange -Input [3]: [sum#17, sum#18, sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [3]: [sum#14, sum#15, sum#16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(27) HashAggregate [codegen id : 8] -Input [3]: [sum#17, sum#18, sum#19] +(24) CometHashAggregate +Input [3]: [sum#14, sum#15, sum#16] Keys: [] Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22] -Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20 AS store_only#23, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21 AS catalog_only#24, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22 AS store_and_catalog#25] + +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/extended.txt new file mode 100644 index 0000000000..07252f3fb5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/extended.txt @@ -0,0 +1,30 @@ +CometColumnarToRow ++- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometNativeScan parquet spark_catalog.default.catalog_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 27 out of 27 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/simplified.txt index cc1e94c5a1..18199546d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/simplified.txt @@ -1,44 +1,27 @@ -WholeStageCodegen (8) - HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (7) - HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum] - Project [customer_sk,customer_sk] - SortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [customer_sk,item_sk] - HashAggregate [ss_customer_sk,ss_item_sk] [customer_sk,item_sk] - InputAdapter - Exchange [ss_customer_sk,ss_item_sk] #2 - WholeStageCodegen (2) - HashAggregate [ss_customer_sk,ss_item_sk] - Project [ss_item_sk,ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - WholeStageCodegen (6) - Sort [customer_sk,item_sk] - HashAggregate [cs_bill_customer_sk,cs_item_sk] [customer_sk,item_sk] - InputAdapter - Exchange [cs_bill_customer_sk,cs_item_sk] #4 - WholeStageCodegen (5) - HashAggregate [cs_bill_customer_sk,cs_item_sk] - Project [cs_bill_customer_sk,cs_item_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 + CometHashAggregate [sum,sum,sum] [store_only,catalog_only,store_and_catalog,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] + CometExchange #1 + CometHashAggregate [customer_sk,customer_sk] [sum,sum,sum] + CometProject [customer_sk,customer_sk] + CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] + CometExchange [ss_customer_sk,ss_item_sk] #2 + CometHashAggregate [ss_customer_sk,ss_item_sk] + CometProject [ss_item_sk,ss_customer_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] + CometExchange [cs_bill_customer_sk,cs_item_sk] #4 + CometHashAggregate [cs_bill_customer_sk,cs_item_sk] + CometProject [cs_bill_customer_sk,cs_item_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/explain.txt index 340150a42d..4966606a2a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/explain.txt @@ -1,150 +1,145 @@ == Physical Plan == -* Project (26) -+- * Sort (25) - +- Exchange (24) - +- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (26) ++- CometProject (25) + +- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] + +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(23) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(24) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(25) Sort [codegen id : 7] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], true, 0 +(25) CometProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] -(26) Project [codegen id : 7] -Output [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] +(26) CometColumnarToRow [codegen id : 3] +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/extended.txt new file mode 100644 index 0000000000..41355f4433 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometProject + +- CometSort + +- CometColumnarExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 22 out of 24 eligible operators (91%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/simplified.txt index b831f96b22..a848d89c1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/simplified.txt @@ -1,42 +1,32 @@ -WholeStageCodegen (7) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] - Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio] - InputAdapter - Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (6) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] +WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] + CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 + WholeStageCodegen (2) + Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometExchange [i_class] #2 + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/explain.txt index 18c9bd3425..3447b6efa3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/explain.txt @@ -1,187 +1,168 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.ship_mode (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.call_center (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometNativeScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.call_center (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometNativeScan parquet spark_catalog.default.date_dim (19) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_warehouse_sk#4] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [cs_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight -(9) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] +(7) CometProject Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -(10) Scan parquet spark_catalog.default.ship_mode +(8) CometNativeScan parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [sm_ship_mode_sk#8, sm_type#9] Condition : isnotnull(sm_ship_mode_sk#8) -(13) BroadcastExchange +(10) CometProject Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] + +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_ship_mode_sk#3] -Right keys [1]: [sm_ship_mode_sk#8] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight -(15) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] +(13) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#10, cc_name#11] +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#11, cc_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#10, cc_name#11] - -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#10, cc_name#11] -Condition : isnotnull(cc_call_center_sk#10) +(15) CometFilter +Input [2]: [cc_call_center_sk#11, cc_name#12] +Condition : isnotnull(cc_call_center_sk#11) -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cc_call_center_sk#11, cc_name#12] -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_call_center_sk#2] -Right keys [1]: [cc_call_center_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cs_call_center_sk#2], [cc_call_center_sk#11], Inner, BuildRight -(21) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] +(18) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_call_center_sk#11, cc_name#12] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] +(19) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) -(24) Filter [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] -Input [2]: [d_date_sk#12, d_month_seq#13] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(26) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(24) CometProject +Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12, d_date_sk#13] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] -(28) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] - -(29) HashAggregate [codegen id : 5] -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(25) CometHashAggregate +Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(30) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 6] -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(32) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,cc_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,cc_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/extended.txt new file mode 100644 index 0000000000..c075163849 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/extended.txt @@ -0,0 +1,31 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.ship_mode + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 28 out of 28 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/simplified.txt index db126ff908..98a215dbc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/simplified.txt @@ -1,48 +1,31 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,w_warehouse_name] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] - BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk] - Project [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_ship_mode_sk,cs_call_center_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,cc_name] #1 + CometHashAggregate [cs_ship_date_sk,cs_sold_date_sk] [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] + CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] + CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometNativeScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [cc_call_center_sk,cc_name] #4 + CometFilter [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/explain.txt index 70584d3db8..8783db0c00 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/explain.txt @@ -1,60 +1,70 @@ == Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometHashAggregate (37) - +- CometExchange (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) :- CometProject (29) : +- CometBroadcastHashJoin (28) : :- CometProject (23) : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) + : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (12) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (16) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- CometBroadcastExchange (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6, d_year#7, d_moy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] @@ -86,9 +96,12 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(12) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#11] @@ -102,9 +115,12 @@ Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] Arguments: [customer_sk#12], [ws_bill_customer_sk#9 AS customer_sk#12] -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(16) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Arguments: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#15] @@ -135,9 +151,12 @@ Arguments: [c_customer_sk#1], [customer_sk#12], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(24) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_county#18] -Arguments: [ca_address_sk#17, ca_county#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] +ReadSchema: struct (25) CometFilter Input [2]: [ca_address_sk#17, ca_county#18] @@ -160,45 +179,52 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(30) CometNativeScan parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (31) CometFilter Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Condition : isnotnull(cd_demo_sk#19) -(32) CometBroadcastExchange +(32) CometProject Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#20, 1, true, false, true) AS cd_gender#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) AS cd_education_status#30, cd_purchase_estimate#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#24, 10, true, false, true) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(33) CometBroadcastHashJoin +(33) CometBroadcastExchange +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(34) CometBroadcastHashJoin Left output [1]: [c_current_cdemo_sk#2] -Right output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Right output [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight -(34) CometProject -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(35) CometProject +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -(35) CometHashAggregate -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(36) CometHashAggregate +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [partial_count(1)] -(36) CometExchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(37) CometExchange +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(37) CometHashAggregate -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(38) CometHashAggregate +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [count(1)] -(38) CometTakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#20 ASC NULLS FIRST,cd_marital_status#21 ASC NULLS FIRST,cd_education_status#22 ASC NULLS FIRST,cd_purchase_estimate#23 ASC NULLS FIRST,cd_credit_rating#24 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[cd_gender#20,cd_marital_status#21,cd_education_status#22,cnt1#29,cd_purchase_estimate#23,cnt2#30,cd_credit_rating#24,cnt3#31,cd_dep_count#25,cnt4#32,cd_dep_employed_count#26,cnt5#33,cd_dep_college_count#27,cnt6#34]), [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34], 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34] +(39) CometTakeOrderedAndProject +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#28 ASC NULLS FIRST,cd_marital_status#29 ASC NULLS FIRST,cd_education_status#30 ASC NULLS FIRST,cd_purchase_estimate#23 ASC NULLS FIRST,cd_credit_rating#31 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[cd_gender#28,cd_marital_status#29,cd_education_status#30,cnt1#33,cd_purchase_estimate#23,cnt2#34,cd_credit_rating#31,cnt3#35,cd_dep_count#25,cnt4#36,cd_dep_employed_count#26,cnt5#37,cd_dep_college_count#27,cnt6#38]), [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38], 100, 0, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] -(39) ColumnarToRow [codegen id : 1] -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34] +(40) CometColumnarToRow [codegen id : 1] +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/extended.txt new file mode 100644 index 0000000000..42c478c7c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/simplified.txt index 291f8cde65..837c58b9aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] - CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count,count(1)] + CometHashAggregate [count] [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count(1)] CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] @@ -13,29 +13,30 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [customer_sk] #4 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/explain.txt index 81a4d45adf..4f4c84d308 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/explain.txt @@ -1,326 +1,383 @@ == Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometProject (60) - +- CometBroadcastHashJoin (59) - :- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (44) - : +- CometFilter (43) - : +- CometHashAggregate (42) - : +- CometExchange (41) - : +- CometHashAggregate (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometFilter (33) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (32) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- CometBroadcastExchange (58) - +- CometHashAggregate (57) - +- CometExchange (56) - +- CometHashAggregate (55) - +- CometProject (54) - +- CometBroadcastHashJoin (53) - :- CometProject (51) - : +- CometBroadcastHashJoin (50) - : :- CometFilter (48) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (47) - : +- ReusedExchange (49) - +- ReusedExchange (52) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(13) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(14) CometExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(16) CometFilter -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Arguments: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] - -(18) CometFilter -Input [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_customer_id#19)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Right output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_sk#18], [ss_customer_sk#26], Inner, BuildRight - -(21) CometProject -Input [12]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(23) CometFilter -Input [2]: [d_date_sk#30, d_year#31] -Condition : ((isnotnull(d_year#31) AND (d_year#31 = 2002)) AND isnotnull(d_date_sk#30)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(25) CometBroadcastHashJoin -Left output [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Right output [2]: [d_date_sk#30, d_year#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight - -(26) CometProject -Input [12]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#30, d_year#31] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] - -(27) CometHashAggregate -Input [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(28) CometExchange -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Arguments: hashpartitioning(c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(29) CometHashAggregate -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(30) CometBroadcastExchange -Input [5]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Arguments: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#16, year_total#17] -Right output [5]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Arguments: [customer_id#16], [customer_id#33], Inner, BuildRight - -(32) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Arguments: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] - -(33) CometFilter -Input [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Condition : (isnotnull(c_customer_sk#38) AND isnotnull(c_customer_id#39)) - -(34) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) + +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] + +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] + +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight + +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] + +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight + +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] -(35) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Right output [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Arguments: [c_customer_sk#38], [ws_bill_customer_sk#46], Inner, BuildRight +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(35) CometFilter +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) (36) CometProject -Input [12]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Arguments: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49], [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] - -(37) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#50, d_year#51] - -(38) CometBroadcastHashJoin -Left output [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Right output [2]: [d_date_sk#50, d_year#51] -Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight - -(39) CometProject -Input [12]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49, d_date_sk#50, d_year#51] -Arguments: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51], [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51] - -(40) CometHashAggregate -Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51] -Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] - -(41) CometExchange -Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#52] -Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(42) CometHashAggregate -Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#52] -Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] - -(43) CometFilter -Input [2]: [customer_id#53, year_total#54] -Condition : (isnotnull(year_total#54) AND (year_total#54 > 0.00)) - -(44) CometBroadcastExchange -Input [2]: [customer_id#53, year_total#54] -Arguments: [customer_id#53, year_total#54] - -(45) CometBroadcastHashJoin -Left output [7]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Right output [2]: [customer_id#53, year_total#54] -Arguments: [customer_id#16], [customer_id#53], Inner, BuildRight - -(46) CometProject -Input [9]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, customer_id#53, year_total#54] -Arguments: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54], [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54] - -(47) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Arguments: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] + +(37) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(38) CometFilter +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_bill_customer_sk#58) + +(39) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight + +(41) CometProject +Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#62, d_year#63] + +(43) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Right output [2]: [d_date_sk#62, d_year#63] +Arguments: [ws_sold_date_sk#61], [d_date_sk#62], Inner, BuildRight + +(44) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#62, d_year#63] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] + +(45) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] + +(46) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] (48) CometFilter -Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Condition : (isnotnull(c_customer_sk#55) AND isnotnull(c_customer_id#56)) +Input [2]: [customer_id#65, year_total#66] +Condition : (isnotnull(year_total#66) AND (year_total#66 > 0.00)) -(49) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +(49) CometBroadcastExchange +Input [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#65, year_total#66] (50) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Right output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Arguments: [c_customer_sk#55], [ws_bill_customer_sk#63], Inner, BuildRight +Left output [7]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Right output [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#22], [customer_id#65], Inner, BuildRight (51) CometProject -Input [12]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Arguments: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66], [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Input [9]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, customer_id#65, year_total#66] +Arguments: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66], [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] -(52) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#67, d_year#68] +(52) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(53) CometBroadcastHashJoin -Left output [10]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Right output [2]: [d_date_sk#67, d_year#68] -Arguments: [ws_sold_date_sk#66], [d_date_sk#67], Inner, BuildRight +(53) CometFilter +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Condition : (isnotnull(c_customer_sk#67) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true))) (54) CometProject -Input [12]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66, d_date_sk#67, d_year#68] -Arguments: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68], [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Arguments: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57], [c_customer_sk#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#69, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#70, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#71, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#73, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#74, 50, true, false, true) AS c_email_address#57] -(55) CometHashAggregate -Input [10]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] -Keys [8]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] +(55) ReusedExchange [Reuses operator id: 39] +Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(56) CometExchange -Input [9]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, sum#69] -Arguments: hashpartitioning(c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_sk#67], [ws_bill_customer_sk#75], Inner, BuildRight -(57) CometHashAggregate -Input [9]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, sum#69] -Keys [8]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] +(57) CometProject +Input [12]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(58) CometBroadcastExchange -Input [2]: [customer_id#70, year_total#71] -Arguments: [customer_id#70, year_total#71] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#79, d_year#80] (59) CometBroadcastHashJoin -Left output [8]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54] -Right output [2]: [customer_id#70, year_total#71] -Arguments: [customer_id#16], [customer_id#70], Inner, (CASE WHEN (year_total#54 > 0.00) THEN (year_total#71 / year_total#54) ELSE 0E-20 END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#37 / year_total#17) ELSE 0E-20 END), BuildRight +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [ws_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight (60) CometProject -Input [10]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54, customer_id#70, year_total#71] -Arguments: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36], [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] - -(61) CometTakeOrderedAndProject -Input [4]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#33 ASC NULLS FIRST,customer_first_name#34 ASC NULLS FIRST,customer_last_name#35 ASC NULLS FIRST,customer_email_address#36 ASC NULLS FIRST], output=[customer_id#33,customer_first_name#34,customer_last_name#35,customer_email_address#36]), [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36], 100, [customer_id#33 ASC NULLS FIRST, customer_first_name#34 ASC NULLS FIRST, customer_last_name#35 ASC NULLS FIRST, customer_email_address#36 ASC NULLS FIRST], [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] - -(62) ColumnarToRow [codegen id : 1] -Input [4]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] + +(61) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] + +(62) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#82, year_total#83] + +(65) CometBroadcastHashJoin +Left output [8]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] +Right output [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#22], [customer_id#82], Inner, (CASE WHEN (year_total#66 > 0.00) THEN (year_total#83 / year_total#66) ELSE 0E-20 END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#43 / year_total#23) ELSE 0E-20 END), BuildRight + +(66) CometProject +Input [10]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66, customer_id#82, year_total#83] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] + +(67) CometTakeOrderedAndProject +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_email_address#42 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_email_address#42]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], 100, 0, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_email_address#42 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] + +(68) CometColumnarToRow [codegen id : 1] +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/extended.txt new file mode 100644 index 0000000000..bcd6c3e73d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 75 out of 75 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/simplified.txt index 42f117f1cb..21abb3ad03 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] CometProject [customer_id,customer_first_name,customer_last_name,customer_email_address] @@ -8,57 +8,63 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 - CometBroadcastExchange [customer_id,year_total] #9 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #10 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/explain.txt index b5331eee1f..39f1048822 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/explain.txt @@ -1,116 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/extended.txt new file mode 100644 index 0000000000..9b405bc9ee --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/simplified.txt index a438deae67..33747406ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] CometExchange [i_class] #1 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/explain.txt index d424b863f8..f0c798a4b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/explain.txt @@ -1,25 +1,25 @@ == Physical Plan == -* ColumnarToRow (77) -+- CometTakeOrderedAndProject (76) - +- CometBroadcastHashJoin (75) - :- CometFilter (56) - : +- CometHashAggregate (55) - : +- CometExchange (54) - : +- CometHashAggregate (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) @@ -28,99 +28,128 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - +- CometBroadcastExchange (74) - +- CometFilter (73) - +- CometHashAggregate (72) - +- CometExchange (71) - +- CometHashAggregate (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometBroadcastHashJoin (60) - : : :- CometFilter (58) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (57) - : : +- ReusedExchange (59) - : +- ReusedExchange (61) - +- CometBroadcastExchange (67) - +- CometProject (66) - +- CometFilter (65) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (64) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometNativeScan: `spark_catalog`.`default`.`item` +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometNativeScan: `spark_catalog`.`default`.`item` +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] @@ -139,9 +168,12 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [d_date_sk#21, d_year#22] @@ -212,331 +244,396 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(34) CometBroadcastHashJoin +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(35) CometBroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [brand_id#24, class_id#25, category_id#26] -(36) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Right output [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(37) CometProject +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] -(39) CometBroadcastHashJoin +(47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(48) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : (((isnotnull(i_item_sk#31) AND isnotnull(i_brand_id#32)) AND isnotnull(i_class_id#33)) AND isnotnull(i_category_id#34)) +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(45) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35, d_week_seq#36] +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#39, d_week_seq#40] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(48) CometFilter -Input [2]: [d_date_sk#35, d_week_seq#36] -Condition : ((isnotnull(d_week_seq#36) AND (d_week_seq#36 = Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#35)) +(56) CometFilter +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) -(49) CometProject -Input [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35], [d_date_sk#35] +(57) CometProject +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -(56) CometFilter -Input [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Condition : (isnotnull(sales#43) AND (cast(sales#43 as decimal(32,6)) > cast(Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(57) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Arguments: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] +(65) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct -(58) CometFilter -Input [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Condition : isnotnull(ss_item_sk#47) +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) -(59) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#51] +(67) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#55] -(60) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [1]: [ss_item_sk#51] -Arguments: [ss_item_sk#47], [ss_item_sk#51], LeftSemi, BuildRight +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight -(61) ReusedExchange [Reuses operator id: 44] -Output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] -(62) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_item_sk#47], [i_item_sk#52], Inner, BuildRight +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight -(63) CometProject -Input [8]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] -(64) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56, d_week_seq#57] +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#60, d_week_seq#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(65) CometFilter -Input [2]: [d_date_sk#56, d_week_seq#57] -Condition : ((isnotnull(d_week_seq#57) AND (d_week_seq#57 = Subquery scalar-subquery#58, [id=#59])) AND isnotnull(d_date_sk#56)) +(73) CometFilter +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) -(66) CometProject -Input [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56], [d_date_sk#56] +(74) CometProject +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] -(67) CometBroadcastExchange -Input [1]: [d_date_sk#56] -Arguments: [d_date_sk#56] +(75) CometBroadcastExchange +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] -(68) CometBroadcastHashJoin -Left output [6]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] -Right output [1]: [d_date_sk#56] -Arguments: [ss_sold_date_sk#50], [d_date_sk#56], Inner, BuildRight +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight -(69) CometProject -Input [7]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55, d_date_sk#56] -Arguments: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] -(70) CometHashAggregate -Input [5]: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [partial_sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), partial_count(1)] +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] -(71) CometExchange -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Arguments: hashpartitioning(i_brand_id#53, i_class_id#54, i_category_id#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(79) CometExchange +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(72) CometHashAggregate -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), count(1)] +(80) CometHashAggregate +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] -(73) CometFilter -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) +(81) CometFilter +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(74) CometBroadcastExchange -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] +(82) CometBroadcastExchange +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -(75) CometBroadcastHashJoin -Left output [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Right output [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [i_brand_id#32, i_class_id#33, i_category_id#34], [i_brand_id#53, i_class_id#54, i_category_id#55], Inner, BuildRight +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight -(76) CometTakeOrderedAndProject -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#32 ASC NULLS FIRST,i_class_id#33 ASC NULLS FIRST,i_category_id#34 ASC NULLS FIRST], output=[channel#42,i_brand_id#32,i_class_id#33,i_category_id#34,sales#43,number_sales#44,channel#63,i_brand_id#53,i_class_id#54,i_category_id#55,sales#64,number_sales#65]), [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65], 100, [i_brand_id#32 ASC NULLS FIRST, i_class_id#33 ASC NULLS FIRST, i_category_id#34 ASC NULLS FIRST], [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -(77) ColumnarToRow [codegen id : 1] -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* ColumnarToRow (94) -+- CometHashAggregate (93) - +- CometExchange (92) - +- CometHashAggregate (91) - +- CometUnion (90) - :- CometProject (81) - : +- CometBroadcastHashJoin (80) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (78) - : +- ReusedExchange (79) - :- CometProject (85) - : +- CometBroadcastHashJoin (84) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (82) - : +- ReusedExchange (83) - +- CometProject (89) - +- CometBroadcastHashJoin (88) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (86) - +- ReusedExchange (87) - - -(78) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Arguments: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] - -(79) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#69] - -(80) CometBroadcastHashJoin -Left output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Right output [1]: [d_date_sk#69] -Arguments: [ss_sold_date_sk#68], [d_date_sk#69], Inner, BuildRight - -(81) CometProject -Input [4]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68, d_date_sk#69] -Arguments: [quantity#70, list_price#71], [ss_quantity#66 AS quantity#70, ss_list_price#67 AS list_price#71] - -(82) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Arguments: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] - -(83) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#75] - -(84) CometBroadcastHashJoin -Left output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Right output [1]: [d_date_sk#75] -Arguments: [cs_sold_date_sk#74], [d_date_sk#75], Inner, BuildRight - -(85) CometProject -Input [4]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74, d_date_sk#75] -Arguments: [quantity#76, list_price#77], [cs_quantity#72 AS quantity#76, cs_list_price#73 AS list_price#77] - -(86) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Arguments: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometNativeScan parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometNativeScan parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] +ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#73] (88) CometBroadcastHashJoin -Left output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#80], [d_date_sk#81], Inner, BuildRight +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight (89) CometProject -Input [4]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80, d_date_sk#81] -Arguments: [quantity#82, list_price#83], [ws_quantity#78 AS quantity#82, ws_list_price#79 AS list_price#83] +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] + +(90) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] +ReadSchema: struct + +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#79] + +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(93) CometProject +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] + +(94) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] +ReadSchema: struct + +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#85] + +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight -(90) CometUnion -Child 0 Input [2]: [quantity#70, list_price#71] -Child 1 Input [2]: [quantity#76, list_price#77] -Child 2 Input [2]: [quantity#82, list_price#83] +(97) CometProject +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] + +(98) CometUnion +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] -(91) CometHashAggregate -Input [2]: [quantity#70, list_price#71] +(99) CometHashAggregate +Input [2]: [quantity#74, list_price#75] Keys: [] -Functions [1]: [partial_avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] -(92) CometExchange -Input [2]: [sum#84, count#85] +(100) CometExchange +Input [2]: [sum#88, count#89] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(93) CometHashAggregate -Input [2]: [sum#84, count#85] +(101) CometHashAggregate +Input [2]: [sum#88, count#89] Keys: [] -Functions [1]: [avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] -(94) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#86] +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#90] -Subquery:2 Hosting operator id = 48 Hosting Expression = Subquery scalar-subquery#37, [id=#38] -* ColumnarToRow (98) -+- CometProject (97) - +- CometFilter (96) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (95) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(95) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] +(103) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] +ReadSchema: struct -(96) CometFilter -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Condition : (((((isnotnull(d_year#88) AND isnotnull(d_moy#89)) AND isnotnull(d_dom#90)) AND (d_year#88 = 1999)) AND (d_moy#89 = 12)) AND (d_dom#90 = 16)) +(104) CometFilter +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1999)) AND (d_moy#93 = 12)) AND (d_dom#94 = 16)) -(97) CometProject -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87], [d_week_seq#87] +(105) CometProject +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] -(98) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#87] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#91] -Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#58, [id=#59] -* ColumnarToRow (102) -+- CometProject (101) - +- CometFilter (100) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (99) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (107) -(99) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +(107) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] +ReadSchema: struct -(100) CometFilter -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1998)) AND (d_moy#93 = 12)) AND (d_dom#94 = 16)) +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1998)) AND (d_moy#97 = 12)) AND (d_dom#98 = 16)) -(101) CometProject -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91], [d_week_seq#91] +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] -(102) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#91] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/extended.txt new file mode 100644 index 0000000000..fc08251119 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/extended.txt @@ -0,0 +1,290 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometBroadcastHashJoin + :- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : +- Subquery + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 280 out of 284 eligible operators (98%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/simplified.txt index d5143df8c8..92429c0948 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/simplified.txt @@ -1,44 +1,44 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #2 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #11 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #8 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -49,66 +49,74 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #8 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #6 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #2 - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #12 + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #13 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [ss_item_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] Subquery #3 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/explain.txt index a0aa70424a..ec2e42fbd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/explain.txt @@ -1,32 +1,32 @@ == Physical Plan == -* ColumnarToRow (92) -+- CometTakeOrderedAndProject (91) - +- CometHashAggregate (90) - +- CometExchange (89) - +- CometHashAggregate (88) - +- CometUnion (87) - :- CometHashAggregate (66) - : +- CometExchange (65) - : +- CometHashAggregate (64) - : +- CometUnion (63) - : :- CometFilter (56) - : : +- CometHashAggregate (55) - : : +- CometExchange (54) - : : +- CometHashAggregate (53) - : : +- CometProject (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometBroadcastHashJoin (39) +* CometColumnarToRow (122) ++- CometTakeOrderedAndProject (121) + +- CometHashAggregate (120) + +- CometExchange (119) + +- CometHashAggregate (118) + +- CometUnion (117) + :- CometHashAggregate (96) + : +- CometExchange (95) + : +- CometHashAggregate (94) + : +- CometUnion (93) + : :- CometFilter (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometBroadcastHashJoin (47) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometBroadcastExchange (38) - : : : : +- CometProject (37) - : : : : +- CometBroadcastHashJoin (36) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (46) + : : : : +- CometProject (45) + : : : : +- CometBroadcastHashJoin (44) : : : : :- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : : +- CometBroadcastExchange (35) - : : : : +- CometBroadcastHashJoin (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : : +- CometBroadcastExchange (43) + : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) : : : : : +- CometExchange (31) : : : : : +- CometHashAggregate (30) @@ -35,107 +35,158 @@ : : : : : :- CometProject (26) : : : : : : +- CometBroadcastHashJoin (25) : : : : : : :- CometFilter (6) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) : : : : : : +- CometBroadcastExchange (24) : : : : : : +- CometBroadcastHashJoin (23) : : : : : : :- CometFilter (8) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) : : : : : : +- CometBroadcastExchange (22) : : : : : : +- CometProject (21) : : : : : : +- CometBroadcastHashJoin (20) : : : : : : :- CometProject (15) : : : : : : : +- CometBroadcastHashJoin (14) : : : : : : : :- CometFilter (10) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) : : : : : : : +- CometBroadcastExchange (13) : : : : : : : +- CometFilter (12) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) : : : : : : +- CometBroadcastExchange (19) : : : : : : +- CometProject (18) : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) : : : : : +- ReusedExchange (27) - : : : : +- ReusedExchange (33) - : : : +- CometBroadcastExchange (44) - : : : +- CometBroadcastHashJoin (43) - : : : :- CometFilter (41) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : : +- ReusedExchange (42) - : : +- CometBroadcastExchange (50) - : : +- CometProject (49) - : : +- CometFilter (48) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - : :- CometFilter (59) - : : +- CometHashAggregate (58) - : : +- ReusedExchange (57) - : +- CometFilter (62) - : +- CometHashAggregate (61) - : +- ReusedExchange (60) - :- CometHashAggregate (71) - : +- CometExchange (70) - : +- CometHashAggregate (69) - : +- CometHashAggregate (68) - : +- ReusedExchange (67) - :- CometHashAggregate (76) - : +- CometExchange (75) - : +- CometHashAggregate (74) - : +- CometHashAggregate (73) - : +- ReusedExchange (72) - :- CometHashAggregate (81) - : +- CometExchange (80) - : +- CometHashAggregate (79) - : +- CometHashAggregate (78) - : +- ReusedExchange (77) - +- CometHashAggregate (86) - +- CometExchange (85) - +- CometHashAggregate (84) - +- CometHashAggregate (83) - +- ReusedExchange (82) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : : +- CometBroadcastExchange (41) + : : : : +- CometProject (40) + : : : : +- CometBroadcastHashJoin (39) + : : : : :- CometProject (37) + : : : : : +- CometBroadcastHashJoin (36) + : : : : : :- CometFilter (34) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : : +- ReusedExchange (35) + : : : : +- ReusedExchange (38) + : : : +- CometBroadcastExchange (52) + : : : +- CometBroadcastHashJoin (51) + : : : :- CometFilter (49) + : : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : : +- ReusedExchange (50) + : : +- CometBroadcastExchange (58) + : : +- CometProject (57) + : : +- CometFilter (56) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + : :- CometFilter (78) + : : +- CometHashAggregate (77) + : : +- CometExchange (76) + : : +- CometHashAggregate (75) + : : +- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometBroadcastHashJoin (68) + : : : : :- CometFilter (66) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (65) + : : : : +- ReusedExchange (67) + : : : +- ReusedExchange (69) + : : +- ReusedExchange (72) + : +- CometFilter (92) + : +- CometHashAggregate (91) + : +- CometExchange (90) + : +- CometHashAggregate (89) + : +- CometProject (88) + : +- CometBroadcastHashJoin (87) + : :- CometProject (85) + : : +- CometBroadcastHashJoin (84) + : : :- CometBroadcastHashJoin (82) + : : : :- CometFilter (80) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (79) + : : : +- ReusedExchange (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (86) + :- CometHashAggregate (101) + : +- CometExchange (100) + : +- CometHashAggregate (99) + : +- CometHashAggregate (98) + : +- ReusedExchange (97) + :- CometHashAggregate (106) + : +- CometExchange (105) + : +- CometHashAggregate (104) + : +- CometHashAggregate (103) + : +- ReusedExchange (102) + :- CometHashAggregate (111) + : +- CometExchange (110) + : +- CometHashAggregate (109) + : +- CometHashAggregate (108) + : +- ReusedExchange (107) + +- CometHashAggregate (116) + +- CometExchange (115) + +- CometHashAggregate (114) + +- CometHashAggregate (113) + +- ReusedExchange (112) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometNativeScan: `spark_catalog`.`default`.`item` +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct (8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometNativeScan: `spark_catalog`.`default`.`item` +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] @@ -154,9 +205,12 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (17) CometFilter Input [2]: [d_date_sk#21, d_year#22] @@ -227,377 +281,532 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(34) CometBroadcastHashJoin +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(35) CometBroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [brand_id#24, class_id#25, category_id#26] -(36) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Right output [3]: [brand_id#24, class_id#25, category_id#26] Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(37) CometProject +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] -(39) CometBroadcastHashJoin +(47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(48) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : isnotnull(i_item_sk#31) +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(45) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35, d_year#36, d_moy#37] +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct -(48) CometFilter -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2000)) AND (d_moy#37 = 11)) AND isnotnull(d_date_sk#35)) +(56) CometFilter +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) -(49) CometProject -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35], [d_date_sk#35] +(57) CometProject +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -(56) CometFilter -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sales#42, number_sales#43] -Condition : (isnotnull(sales#42) AND (cast(sales#42 as decimal(32,6)) > cast(Subquery scalar-subquery#44, [id=#45] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(65) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(66) CometFilter +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) + +(67) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#54] + +(68) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#54] +Arguments: [cs_item_sk#50], [ss_item_sk#54], LeftSemi, BuildRight + +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] + +(70) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_item_sk#50], [i_item_sk#55], Inner, BuildRight + +(71) CometProject +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] + +(72) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#59] + +(73) CometBroadcastHashJoin +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [cs_sold_date_sk#53], [d_date_sk#59], Inner, BuildRight + +(74) CometProject +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] + +(75) CometHashAggregate +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] + +(76) CometExchange +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(77) CometHashAggregate +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] + +(78) CometFilter +Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(79) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#69)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(80) CometFilter +Input [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Condition : isnotnull(ws_item_sk#66) + +(81) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#70] + +(82) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [1]: [ss_item_sk#70] +Arguments: [ws_item_sk#66], [ss_item_sk#70], LeftSemi, BuildRight + +(83) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_item_sk#66], [i_item_sk#71], Inner, BuildRight + +(85) CometProject +Input [8]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] + +(86) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#75] + +(87) CometBroadcastHashJoin +Left output [6]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] +Right output [1]: [d_date_sk#75] +Arguments: [ws_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight + +(88) CometProject +Input [7]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74, d_date_sk#75] +Arguments: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] + +(89) CometHashAggregate +Input [5]: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [partial_sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), partial_count(1)] + +(90) CometExchange +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Arguments: hashpartitioning(i_brand_id#72, i_class_id#73, i_category_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(91) CometHashAggregate +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), count(1)] + +(92) CometFilter +Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(93) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Child 2 Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] + +(94) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] + +(95) CometExchange +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(96) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(97) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#87, sum#82, isEmpty#83, sum#84] + +(98) CometHashAggregate +Input [7]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#87, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#87] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(99) CometHashAggregate +Input [5]: [channel#45, i_brand_id#85, i_class_id#86, sum_sales#88, number_sales#89] +Keys [3]: [channel#45, i_brand_id#85, i_class_id#86] +Functions [2]: [partial_sum(sum_sales#88), partial_sum(number_sales#89)] + +(100) CometExchange +Input [6]: [channel#45, i_brand_id#85, i_class_id#86, sum#90, isEmpty#91, sum#92] +Arguments: hashpartitioning(channel#45, i_brand_id#85, i_class_id#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(101) CometHashAggregate +Input [6]: [channel#45, i_brand_id#85, i_class_id#86, sum#90, isEmpty#91, sum#92] +Keys [3]: [channel#45, i_brand_id#85, i_class_id#86] +Functions [2]: [sum(sum_sales#88), sum(number_sales#89)] + +(102) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#93, i_class_id#94, i_category_id#95, sum#82, isEmpty#83, sum#84] + +(103) CometHashAggregate +Input [7]: [channel#45, i_brand_id#93, i_class_id#94, i_category_id#95, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#93, i_class_id#94, i_category_id#95] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(104) CometHashAggregate +Input [4]: [channel#45, i_brand_id#93, sum_sales#96, number_sales#97] +Keys [2]: [channel#45, i_brand_id#93] +Functions [2]: [partial_sum(sum_sales#96), partial_sum(number_sales#97)] + +(105) CometExchange +Input [5]: [channel#45, i_brand_id#93, sum#98, isEmpty#99, sum#100] +Arguments: hashpartitioning(channel#45, i_brand_id#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(106) CometHashAggregate +Input [5]: [channel#45, i_brand_id#93, sum#98, isEmpty#99, sum#100] +Keys [2]: [channel#45, i_brand_id#93] +Functions [2]: [sum(sum_sales#96), sum(number_sales#97)] + +(107) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#101, i_class_id#102, i_category_id#103, sum#82, isEmpty#83, sum#84] + +(108) CometHashAggregate +Input [7]: [channel#45, i_brand_id#101, i_class_id#102, i_category_id#103, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#101, i_class_id#102, i_category_id#103] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(109) CometHashAggregate +Input [3]: [channel#45, sum_sales#104, number_sales#105] +Keys [1]: [channel#45] +Functions [2]: [partial_sum(sum_sales#104), partial_sum(number_sales#105)] -(57) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#49, isEmpty#50, count#51] +(110) CometExchange +Input [4]: [channel#45, sum#106, isEmpty#107, sum#108] +Arguments: hashpartitioning(channel#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(58) CometHashAggregate -Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#49, isEmpty#50, count#51] -Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48] -Functions [2]: [sum((cast(cs_quantity#52 as decimal(10,0)) * cs_list_price#53)), count(1)] +(111) CometHashAggregate +Input [4]: [channel#45, sum#106, isEmpty#107, sum#108] +Keys [1]: [channel#45] +Functions [2]: [sum(sum_sales#104), sum(number_sales#105)] -(59) CometFilter -Input [6]: [channel#54, i_brand_id#46, i_class_id#47, i_category_id#48, sales#55, number_sales#56] -Condition : (isnotnull(sales#55) AND (cast(sales#55 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#44, [id=#45] as decimal(32,6)))) +(112) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#109, i_class_id#110, i_category_id#111, sum#82, isEmpty#83, sum#84] -(60) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#60, isEmpty#61, count#62] +(113) CometHashAggregate +Input [7]: [channel#45, i_brand_id#109, i_class_id#110, i_category_id#111, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#109, i_class_id#110, i_category_id#111] +Functions [2]: [sum(sales#46), sum(number_sales#47)] -(61) CometHashAggregate -Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#60, isEmpty#61, count#62] -Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] -Functions [2]: [sum((cast(ws_quantity#63 as decimal(10,0)) * ws_list_price#64)), count(1)] - -(62) CometFilter -Input [6]: [channel#65, i_brand_id#57, i_class_id#58, i_category_id#59, sales#66, number_sales#67] -Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#44, [id=#45] as decimal(32,6)))) - -(63) CometUnion -Child 0 Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sales#42, number_sales#43] -Child 1 Input [6]: [channel#54, i_brand_id#46, i_class_id#47, i_category_id#48, sales#55, number_sales#56] -Child 2 Input [6]: [channel#65, i_brand_id#57, i_class_id#58, i_category_id#59, sales#66, number_sales#67] - -(64) CometHashAggregate -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sales#42, number_sales#43] -Keys [4]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum(sales#42), partial_sum(number_sales#43)] - -(65) CometExchange -Input [7]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum#68, isEmpty#69, sum#70] -Arguments: hashpartitioning(channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(66) CometHashAggregate -Input [7]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(67) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#73, sum#68, isEmpty#69, sum#70] - -(68) CometHashAggregate -Input [7]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#73, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#73] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(69) CometHashAggregate -Input [5]: [channel#41, i_brand_id#71, i_class_id#72, sum_sales#74, number_sales#75] -Keys [3]: [channel#41, i_brand_id#71, i_class_id#72] -Functions [2]: [partial_sum(sum_sales#74), partial_sum(number_sales#75)] - -(70) CometExchange -Input [6]: [channel#41, i_brand_id#71, i_class_id#72, sum#76, isEmpty#77, sum#78] -Arguments: hashpartitioning(channel#41, i_brand_id#71, i_class_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(71) CometHashAggregate -Input [6]: [channel#41, i_brand_id#71, i_class_id#72, sum#76, isEmpty#77, sum#78] -Keys [3]: [channel#41, i_brand_id#71, i_class_id#72] -Functions [2]: [sum(sum_sales#74), sum(number_sales#75)] - -(72) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#79, i_class_id#80, i_category_id#81, sum#68, isEmpty#69, sum#70] - -(73) CometHashAggregate -Input [7]: [channel#41, i_brand_id#79, i_class_id#80, i_category_id#81, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#79, i_class_id#80, i_category_id#81] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(74) CometHashAggregate -Input [4]: [channel#41, i_brand_id#79, sum_sales#82, number_sales#83] -Keys [2]: [channel#41, i_brand_id#79] -Functions [2]: [partial_sum(sum_sales#82), partial_sum(number_sales#83)] - -(75) CometExchange -Input [5]: [channel#41, i_brand_id#79, sum#84, isEmpty#85, sum#86] -Arguments: hashpartitioning(channel#41, i_brand_id#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(76) CometHashAggregate -Input [5]: [channel#41, i_brand_id#79, sum#84, isEmpty#85, sum#86] -Keys [2]: [channel#41, i_brand_id#79] -Functions [2]: [sum(sum_sales#82), sum(number_sales#83)] - -(77) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#87, i_class_id#88, i_category_id#89, sum#68, isEmpty#69, sum#70] - -(78) CometHashAggregate -Input [7]: [channel#41, i_brand_id#87, i_class_id#88, i_category_id#89, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#87, i_class_id#88, i_category_id#89] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(79) CometHashAggregate -Input [3]: [channel#41, sum_sales#90, number_sales#91] -Keys [1]: [channel#41] -Functions [2]: [partial_sum(sum_sales#90), partial_sum(number_sales#91)] - -(80) CometExchange -Input [4]: [channel#41, sum#92, isEmpty#93, sum#94] -Arguments: hashpartitioning(channel#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(81) CometHashAggregate -Input [4]: [channel#41, sum#92, isEmpty#93, sum#94] -Keys [1]: [channel#41] -Functions [2]: [sum(sum_sales#90), sum(number_sales#91)] - -(82) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#95, i_class_id#96, i_category_id#97, sum#68, isEmpty#69, sum#70] - -(83) CometHashAggregate -Input [7]: [channel#41, i_brand_id#95, i_class_id#96, i_category_id#97, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#95, i_class_id#96, i_category_id#97] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(84) CometHashAggregate -Input [2]: [sum_sales#98, number_sales#99] +(114) CometHashAggregate +Input [2]: [sum_sales#112, number_sales#113] Keys: [] -Functions [2]: [partial_sum(sum_sales#98), partial_sum(number_sales#99)] +Functions [2]: [partial_sum(sum_sales#112), partial_sum(number_sales#113)] -(85) CometExchange -Input [3]: [sum#100, isEmpty#101, sum#102] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(115) CometExchange +Input [3]: [sum#114, isEmpty#115, sum#116] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(86) CometHashAggregate -Input [3]: [sum#100, isEmpty#101, sum#102] +(116) CometHashAggregate +Input [3]: [sum#114, isEmpty#115, sum#116] Keys: [] -Functions [2]: [sum(sum_sales#98), sum(number_sales#99)] - -(87) CometUnion -Child 0 Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Child 1 Input [6]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#105, sum(sum_sales)#106, sum(number_sales)#107] -Child 2 Input [6]: [channel#41, i_brand_id#79, i_class_id#108, i_category_id#109, sum(sum_sales)#110, sum(number_sales)#111] -Child 3 Input [6]: [channel#41, i_brand_id#112, i_class_id#113, i_category_id#114, sum(sum_sales)#115, sum(number_sales)#116] -Child 4 Input [6]: [channel#117, i_brand_id#118, i_class_id#119, i_category_id#120, sum(sum_sales)#121, sum(number_sales)#122] - -(88) CometHashAggregate -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Keys [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] +Functions [2]: [sum(sum_sales#112), sum(number_sales#113)] + +(117) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Child 1 Input [6]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#119, sum(sum_sales)#120, sum(number_sales)#121] +Child 2 Input [6]: [channel#45, i_brand_id#93, i_class_id#122, i_category_id#123, sum(sum_sales)#124, sum(number_sales)#125] +Child 3 Input [6]: [channel#45, i_brand_id#126, i_class_id#127, i_category_id#128, sum(sum_sales)#129, sum(number_sales)#130] +Child 4 Input [6]: [channel#131, i_brand_id#132, i_class_id#133, i_category_id#134, sum(sum_sales)#135, sum(number_sales)#136] + +(118) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] Functions: [] -(89) CometExchange -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Arguments: hashpartitioning(channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(119) CometExchange +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(90) CometHashAggregate -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Keys [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] +(120) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] Functions: [] -(91) CometTakeOrderedAndProject -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#41 ASC NULLS FIRST,i_brand_id#32 ASC NULLS FIRST,i_class_id#33 ASC NULLS FIRST,i_category_id#34 ASC NULLS FIRST], output=[channel#41,i_brand_id#32,i_class_id#33,i_category_id#34,sum_sales#103,number_sales#104]), [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104], 100, [channel#41 ASC NULLS FIRST, i_brand_id#32 ASC NULLS FIRST, i_class_id#33 ASC NULLS FIRST, i_category_id#34 ASC NULLS FIRST], [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] +(121) CometTakeOrderedAndProject +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#117,number_sales#118]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] -(92) ColumnarToRow [codegen id : 1] -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] +(122) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] ===== Subqueries ===== -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#44, [id=#45] -* ColumnarToRow (112) -+- CometHashAggregate (111) - +- CometExchange (110) - +- CometHashAggregate (109) - +- CometUnion (108) - :- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (93) - : +- ReusedExchange (94) - :- CometProject (103) - : +- CometBroadcastHashJoin (102) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (97) - : +- CometBroadcastExchange (101) - : +- CometProject (100) - : +- CometFilter (99) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (98) - +- CometProject (107) - +- CometBroadcastHashJoin (106) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (104) - +- ReusedExchange (105) - - -(93) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125] -Arguments: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125] - -(94) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#126] - -(95) CometBroadcastHashJoin -Left output [3]: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125] -Right output [1]: [d_date_sk#126] -Arguments: [ss_sold_date_sk#125], [d_date_sk#126], Inner, BuildRight - -(96) CometProject -Input [4]: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125, d_date_sk#126] -Arguments: [quantity#127, list_price#128], [ss_quantity#123 AS quantity#127, ss_list_price#124 AS list_price#128] - -(97) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131] -Arguments: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131] - -(98) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#132, d_year#133] -Arguments: [d_date_sk#132, d_year#133] - -(99) CometFilter -Input [2]: [d_date_sk#132, d_year#133] -Condition : (((isnotnull(d_year#133) AND (d_year#133 >= 1998)) AND (d_year#133 <= 2000)) AND isnotnull(d_date_sk#132)) - -(100) CometProject -Input [2]: [d_date_sk#132, d_year#133] -Arguments: [d_date_sk#132], [d_date_sk#132] - -(101) CometBroadcastExchange -Input [1]: [d_date_sk#132] -Arguments: [d_date_sk#132] - -(102) CometBroadcastHashJoin -Left output [3]: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131] -Right output [1]: [d_date_sk#132] -Arguments: [cs_sold_date_sk#131], [d_date_sk#132], Inner, BuildRight - -(103) CometProject -Input [4]: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131, d_date_sk#132] -Arguments: [quantity#134, list_price#135], [cs_quantity#129 AS quantity#134, cs_list_price#130 AS list_price#135] - -(104) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] -Arguments: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] - -(105) ReusedExchange [Reuses operator id: 101] -Output [1]: [d_date_sk#139] - -(106) CometBroadcastHashJoin -Left output [3]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] -Right output [1]: [d_date_sk#139] -Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight - -(107) CometProject -Input [4]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138, d_date_sk#139] -Arguments: [quantity#140, list_price#141], [ws_quantity#136 AS quantity#140, ws_list_price#137 AS list_price#141] - -(108) CometUnion -Child 0 Input [2]: [quantity#127, list_price#128] -Child 1 Input [2]: [quantity#134, list_price#135] -Child 2 Input [2]: [quantity#140, list_price#141] - -(109) CometHashAggregate -Input [2]: [quantity#127, list_price#128] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] +* CometColumnarToRow (142) ++- CometHashAggregate (141) + +- CometExchange (140) + +- CometHashAggregate (139) + +- CometUnion (138) + :- CometProject (126) + : +- CometBroadcastHashJoin (125) + : :- CometNativeScan parquet spark_catalog.default.store_sales (123) + : +- ReusedExchange (124) + :- CometProject (133) + : +- CometBroadcastHashJoin (132) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (127) + : +- CometBroadcastExchange (131) + : +- CometProject (130) + : +- CometFilter (129) + : +- CometNativeScan parquet spark_catalog.default.date_dim (128) + +- CometProject (137) + +- CometBroadcastHashJoin (136) + :- CometNativeScan parquet spark_catalog.default.web_sales (134) + +- ReusedExchange (135) + + +(123) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#139)] +ReadSchema: struct + +(124) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#140] + +(125) CometBroadcastHashJoin +Left output [3]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139] +Right output [1]: [d_date_sk#140] +Arguments: [ss_sold_date_sk#139], [d_date_sk#140], Inner, BuildRight + +(126) CometProject +Input [4]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139, d_date_sk#140] +Arguments: [quantity#141, list_price#142], [ss_quantity#137 AS quantity#141, ss_list_price#138 AS list_price#142] + +(127) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#145)] +ReadSchema: struct + +(128) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#146, d_year#147] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(129) CometFilter +Input [2]: [d_date_sk#146, d_year#147] +Condition : (((isnotnull(d_year#147) AND (d_year#147 >= 1998)) AND (d_year#147 <= 2000)) AND isnotnull(d_date_sk#146)) + +(130) CometProject +Input [2]: [d_date_sk#146, d_year#147] +Arguments: [d_date_sk#146], [d_date_sk#146] + +(131) CometBroadcastExchange +Input [1]: [d_date_sk#146] +Arguments: [d_date_sk#146] + +(132) CometBroadcastHashJoin +Left output [3]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145] +Right output [1]: [d_date_sk#146] +Arguments: [cs_sold_date_sk#145], [d_date_sk#146], Inner, BuildRight + +(133) CometProject +Input [4]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145, d_date_sk#146] +Arguments: [quantity#148, list_price#149], [cs_quantity#143 AS quantity#148, cs_list_price#144 AS list_price#149] + +(134) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#150, ws_list_price#151, ws_sold_date_sk#152] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#152)] +ReadSchema: struct + +(135) ReusedExchange [Reuses operator id: 131] +Output [1]: [d_date_sk#153] + +(136) CometBroadcastHashJoin +Left output [3]: [ws_quantity#150, ws_list_price#151, ws_sold_date_sk#152] +Right output [1]: [d_date_sk#153] +Arguments: [ws_sold_date_sk#152], [d_date_sk#153], Inner, BuildRight + +(137) CometProject +Input [4]: [ws_quantity#150, ws_list_price#151, ws_sold_date_sk#152, d_date_sk#153] +Arguments: [quantity#154, list_price#155], [ws_quantity#150 AS quantity#154, ws_list_price#151 AS list_price#155] + +(138) CometUnion +Child 0 Input [2]: [quantity#141, list_price#142] +Child 1 Input [2]: [quantity#148, list_price#149] +Child 2 Input [2]: [quantity#154, list_price#155] + +(139) CometHashAggregate +Input [2]: [quantity#141, list_price#142] Keys: [] -Functions [1]: [partial_avg((cast(quantity#127 as decimal(10,0)) * list_price#128))] +Functions [1]: [partial_avg((cast(quantity#141 as decimal(10,0)) * list_price#142))] -(110) CometExchange -Input [2]: [sum#142, count#143] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +(140) CometExchange +Input [2]: [sum#156, count#157] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(111) CometHashAggregate -Input [2]: [sum#142, count#143] +(141) CometHashAggregate +Input [2]: [sum#156, count#157] Keys: [] -Functions [1]: [avg((cast(quantity#127 as decimal(10,0)) * list_price#128))] +Functions [1]: [avg((cast(quantity#141 as decimal(10,0)) * list_price#142))] -(112) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#144] +(142) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#158] -Subquery:2 Hosting operator id = 59 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] -Subquery:3 Hosting operator id = 62 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/extended.txt new file mode 100644 index 0000000000..c24fc36bd9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/extended.txt @@ -0,0 +1,2030 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 2007 out of 2022 eligible operators (99%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/simplified.txt index 9119b51ec8..ff1ccebc20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/simplified.txt @@ -1,54 +1,54 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sales,number_sales] + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #13 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #14 + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #14 + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [d_date_sk] #14 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #15 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #3 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -59,61 +59,91 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #8 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #4 - CometBroadcastExchange [d_date_sk] #12 + CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #3 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #3 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id,i_class_id] #15 - CometHashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id] #18 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] + CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id] #16 - CometHashAggregate [channel,i_brand_id,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,sum_sales,number_sales,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id] #19 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] + CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange [channel] #17 - CometHashAggregate [channel,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,sum_sales,number_sales,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel] #20 + CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] + CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange #18 - CometHashAggregate [sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [sum_sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange #21 + CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/explain.txt index bbf7360e93..c7bca61362 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* ColumnarToRow (141) -+- CometTakeOrderedAndProject (140) - +- CometUnion (139) - :- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (146) ++- CometTakeOrderedAndProject (145) + +- CometUnion (144) + :- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (19) : : : : +- CometBroadcastHashJoin (18) : : : : :- CometProject (14) @@ -18,145 +18,157 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometFilter (16) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (15) - : : : +- CometBroadcastExchange (22) - : : : +- CometFilter (21) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - : : +- CometBroadcastExchange (28) - : : +- CometProject (27) - : : +- CometFilter (26) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (31) - :- CometHashAggregate (63) - : +- CometExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (57) - : : +- CometBroadcastHashJoin (56) - : : :- CometProject (54) - : : : +- CometBroadcastHashJoin (53) - : : : :- CometProject (49) - : : : : +- CometBroadcastHashJoin (48) - : : : : :- CometProject (46) - : : : : : +- CometBroadcastHashJoin (45) - : : : : : :- CometProject (43) - : : : : : : +- CometBroadcastHashJoin (42) - : : : : : : :- CometFilter (40) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : : +- ReusedExchange (41) - : : : : : +- ReusedExchange (44) - : : : : +- ReusedExchange (47) - : : : +- CometBroadcastExchange (52) - : : : +- CometFilter (51) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (50) - : : +- ReusedExchange (55) - : +- ReusedExchange (58) - :- CometHashAggregate (89) - : +- CometExchange (88) - : +- CometHashAggregate (87) - : +- CometProject (86) - : +- CometBroadcastHashJoin (85) - : :- CometProject (83) - : : +- CometBroadcastHashJoin (82) - : : :- CometProject (80) - : : : +- CometBroadcastHashJoin (79) - : : : :- CometProject (74) - : : : : +- CometBroadcastHashJoin (73) - : : : : :- CometProject (71) - : : : : : +- CometBroadcastHashJoin (70) - : : : : : :- CometProject (68) - : : : : : : +- CometBroadcastHashJoin (67) - : : : : : : :- CometFilter (65) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (64) - : : : : : : +- ReusedExchange (66) - : : : : : +- ReusedExchange (69) - : : : : +- ReusedExchange (72) - : : : +- CometBroadcastExchange (78) - : : : +- CometProject (77) - : : : +- CometFilter (76) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (75) - : : +- ReusedExchange (81) - : +- ReusedExchange (84) - :- CometHashAggregate (115) - : +- CometExchange (114) - : +- CometHashAggregate (113) - : +- CometProject (112) - : +- CometBroadcastHashJoin (111) - : :- CometProject (109) - : : +- CometBroadcastHashJoin (108) - : : :- CometProject (106) - : : : +- CometBroadcastHashJoin (105) - : : : :- CometProject (100) - : : : : +- CometBroadcastHashJoin (99) - : : : : :- CometProject (97) - : : : : : +- CometBroadcastHashJoin (96) - : : : : : :- CometProject (94) - : : : : : : +- CometBroadcastHashJoin (93) - : : : : : : :- CometFilter (91) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (90) - : : : : : : +- ReusedExchange (92) - : : : : : +- ReusedExchange (95) - : : : : +- ReusedExchange (98) - : : : +- CometBroadcastExchange (104) - : : : +- CometProject (103) - : : : +- CometFilter (102) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (101) - : : +- ReusedExchange (107) - : +- ReusedExchange (110) - +- CometHashAggregate (138) - +- CometExchange (137) - +- CometHashAggregate (136) - +- CometProject (135) - +- CometBroadcastHashJoin (134) - :- CometProject (132) - : +- CometBroadcastHashJoin (131) - : :- CometProject (129) - : : +- CometBroadcastHashJoin (128) - : : :- CometProject (126) - : : : +- CometBroadcastHashJoin (125) - : : : :- CometProject (123) - : : : : +- CometBroadcastHashJoin (122) - : : : : :- CometProject (120) - : : : : : +- CometBroadcastHashJoin (119) - : : : : : :- CometFilter (117) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (116) - : : : : : +- ReusedExchange (118) - : : : : +- ReusedExchange (121) - : : : +- ReusedExchange (124) - : : +- ReusedExchange (127) - : +- ReusedExchange (130) - +- ReusedExchange (133) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.item (32) + :- CometHashAggregate (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (51) + : : : : +- CometBroadcastHashJoin (50) + : : : : :- CometProject (48) + : : : : : +- CometBroadcastHashJoin (47) + : : : : : :- CometProject (45) + : : : : : : +- CometBroadcastHashJoin (44) + : : : : : : :- CometFilter (42) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (41) + : : : : : : +- ReusedExchange (43) + : : : : : +- ReusedExchange (46) + : : : : +- ReusedExchange (49) + : : : +- CometBroadcastExchange (55) + : : : +- CometProject (54) + : : : +- CometFilter (53) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (52) + : : +- ReusedExchange (58) + : +- ReusedExchange (61) + :- CometHashAggregate (92) + : +- CometExchange (91) + : +- CometHashAggregate (90) + : +- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometProject (86) + : : +- CometBroadcastHashJoin (85) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (77) + : : : : +- CometBroadcastHashJoin (76) + : : : : :- CometProject (74) + : : : : : +- CometBroadcastHashJoin (73) + : : : : : :- CometProject (71) + : : : : : : +- CometBroadcastHashJoin (70) + : : : : : : :- CometFilter (68) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (67) + : : : : : : +- ReusedExchange (69) + : : : : : +- ReusedExchange (72) + : : : : +- ReusedExchange (75) + : : : +- CometBroadcastExchange (81) + : : : +- CometProject (80) + : : : +- CometFilter (79) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (78) + : : +- ReusedExchange (84) + : +- ReusedExchange (87) + :- CometHashAggregate (118) + : +- CometExchange (117) + : +- CometHashAggregate (116) + : +- CometProject (115) + : +- CometBroadcastHashJoin (114) + : :- CometProject (112) + : : +- CometBroadcastHashJoin (111) + : : :- CometProject (109) + : : : +- CometBroadcastHashJoin (108) + : : : :- CometProject (103) + : : : : +- CometBroadcastHashJoin (102) + : : : : :- CometProject (100) + : : : : : +- CometBroadcastHashJoin (99) + : : : : : :- CometProject (97) + : : : : : : +- CometBroadcastHashJoin (96) + : : : : : : :- CometFilter (94) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (93) + : : : : : : +- ReusedExchange (95) + : : : : : +- ReusedExchange (98) + : : : : +- ReusedExchange (101) + : : : +- CometBroadcastExchange (107) + : : : +- CometProject (106) + : : : +- CometFilter (105) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (104) + : : +- ReusedExchange (110) + : +- ReusedExchange (113) + +- CometHashAggregate (143) + +- CometExchange (142) + +- CometHashAggregate (141) + +- CometProject (140) + +- CometBroadcastHashJoin (139) + :- CometProject (135) + : +- CometBroadcastHashJoin (134) + : :- CometProject (132) + : : +- CometBroadcastHashJoin (131) + : : :- CometProject (129) + : : : +- CometBroadcastHashJoin (128) + : : : :- CometProject (126) + : : : : +- CometBroadcastHashJoin (125) + : : : : :- CometProject (123) + : : : : : +- CometBroadcastHashJoin (122) + : : : : : :- CometFilter (120) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (119) + : : : : : +- ReusedExchange (121) + : : : : +- ReusedExchange (124) + : : : +- ReusedExchange (127) + : : +- ReusedExchange (130) + : +- ReusedExchange (133) + +- CometBroadcastExchange (138) + +- CometFilter (137) + +- CometNativeScan parquet spark_catalog.default.item (136) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Arguments: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = M)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#10)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) (5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] @@ -175,9 +187,12 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(9) CometNativeScan: `spark_catalog`.`default`.`customer` +(9) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct (10) CometFilter Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] @@ -200,9 +215,12 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(15) CometNativeScan parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [1]: [cd_demo_sk#19] @@ -221,509 +239,567 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) -(22) CometBroadcastExchange +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24, d_year#25] - -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#24)) - -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] - -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight - -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(31) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] - -(36) CometHashAggregate -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] - -(37) CometExchange -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(26) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) + +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight + +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] (38) CometHashAggregate -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] - -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Arguments: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] - -(40) CometFilter -Input [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Condition : ((isnotnull(cs_bill_cdemo_sk#50) AND isnotnull(cs_bill_customer_sk#49)) AND isnotnull(cs_item_sk#51)) - -(41) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#58, cd_dep_count#59] - -(42) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Right output [2]: [cd_demo_sk#58, cd_dep_count#59] -Arguments: [cs_bill_cdemo_sk#50], [cd_demo_sk#58], Inner, BuildRight - -(43) CometProject -Input [11]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_demo_sk#58, cd_dep_count#59] -Arguments: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59], [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59] - -(44) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] - -(45) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59] -Right output [4]: [c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Arguments: [cs_bill_customer_sk#49], [c_customer_sk#60], Inner, BuildRight - -(46) CometProject -Input [13]: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] - -(47) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#64] - -(48) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Right output [1]: [cd_demo_sk#64] -Arguments: [c_current_cdemo_sk#61], [cd_demo_sk#64], Inner, BuildRight - -(49) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63, cd_demo_sk#64] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63] - -(50) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [ca_address_sk#65, ca_state#66, ca_country#67] - -(51) CometFilter -Input [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Condition : (ca_state#66 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#65)) - -(52) CometBroadcastExchange -Input [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [ca_address_sk#65, ca_state#66, ca_country#67] - -(53) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63] -Right output [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [c_current_addr_sk#62], [ca_address_sk#65], Inner, BuildRight +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(39) CometExchange +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(41) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(42) CometFilter +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) + +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] + +(44) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight + +(45) CometProject +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] + +(46) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] + +(47) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight + +(48) CometProject +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] + +(49) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#70] + +(50) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight + +(51) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] + +(52) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(53) CometFilter +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) (54) CometProject -Input [13]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63, ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73], [ca_address_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) AS ca_state#24, ca_country#73] -(55) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#68] +(55) CometBroadcastExchange +Input [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73] (56) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] -Right output [1]: [d_date_sk#68] -Arguments: [cs_sold_date_sk#57], [d_date_sk#68], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight (57) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67, d_date_sk#68] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] -(58) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#69, i_item_id#70] +(58) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#74] (59) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] -Right output [2]: [i_item_sk#69, i_item_id#70] -Arguments: [cs_item_sk#51], [i_item_sk#69], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [1]: [d_date_sk#74] +Arguments: [cs_sold_date_sk#63], [d_date_sk#74], Inner, BuildRight (60) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67, i_item_sk#69, i_item_id#70] -Arguments: [i_item_id#70, ca_country#67, ca_state#66, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77], [i_item_id#70, ca_country#67, ca_state#66, cast(cs_quantity#52 as decimal(12,2)) AS agg1#71, cast(cs_list_price#53 as decimal(12,2)) AS agg2#72, cast(cs_coupon_amt#55 as decimal(12,2)) AS agg3#73, cast(cs_sales_price#54 as decimal(12,2)) AS agg4#74, cast(cs_net_profit#56 as decimal(12,2)) AS agg5#75, cast(c_birth_year#63 as decimal(12,2)) AS agg6#76, cast(cd_dep_count#59 as decimal(12,2)) AS agg7#77] - -(61) CometHashAggregate -Input [10]: [i_item_id#70, ca_country#67, ca_state#66, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77] -Keys [3]: [i_item_id#70, ca_country#67, ca_state#66] -Functions [7]: [partial_avg(agg1#71), partial_avg(agg2#72), partial_avg(agg3#73), partial_avg(agg4#74), partial_avg(agg5#75), partial_avg(agg6#76), partial_avg(agg7#77)] - -(62) CometExchange -Input [17]: [i_item_id#70, ca_country#67, ca_state#66, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Arguments: hashpartitioning(i_item_id#70, ca_country#67, ca_state#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(63) CometHashAggregate -Input [17]: [i_item_id#70, ca_country#67, ca_state#66, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Keys [3]: [i_item_id#70, ca_country#67, ca_state#66] -Functions [7]: [avg(agg1#71), avg(agg2#72), avg(agg3#73), avg(agg4#74), avg(agg5#75), avg(agg6#76), avg(agg7#77)] - -(64) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Arguments: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] - -(65) CometFilter -Input [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Condition : ((isnotnull(cs_bill_cdemo_sk#93) AND isnotnull(cs_bill_customer_sk#92)) AND isnotnull(cs_item_sk#94)) - -(66) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#101, cd_dep_count#102] - -(67) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Right output [2]: [cd_demo_sk#101, cd_dep_count#102] -Arguments: [cs_bill_cdemo_sk#93], [cd_demo_sk#101], Inner, BuildRight - -(68) CometProject -Input [11]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_demo_sk#101, cd_dep_count#102] -Arguments: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102], [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102] - -(69) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, d_date_sk#74] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] + +(61) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#75, i_item_id#29] + +(62) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [2]: [i_item_sk#75, i_item_id#29] +Arguments: [cs_item_sk#57], [i_item_sk#75], Inner, BuildRight + +(63) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, i_item_sk#75, i_item_id#29] +Arguments: [i_item_id#29, ca_country#73, ca_state#24, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82], [i_item_id#29, ca_country#73, ca_state#24, cast(cs_quantity#58 as decimal(12,2)) AS agg1#76, cast(cs_list_price#59 as decimal(12,2)) AS agg2#77, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#78, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#79, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#80, cast(c_birth_year#69 as decimal(12,2)) AS agg6#81, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#82] + +(64) CometHashAggregate +Input [10]: [i_item_id#29, ca_country#73, ca_state#24, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [partial_avg(agg1#76), partial_avg(agg2#77), partial_avg(agg3#78), partial_avg(agg4#79), partial_avg(agg5#80), partial_avg(agg6#81), partial_avg(agg7#82)] + +(65) CometExchange +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96] +Arguments: hashpartitioning(i_item_id#29, ca_country#73, ca_state#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(66) CometHashAggregate +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [avg(agg1#76), avg(agg2#77), avg(agg3#78), avg(agg4#79), avg(agg5#80), avg(agg6#81), avg(agg7#82)] + +(67) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#105)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(68) CometFilter +Input [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] +Condition : ((isnotnull(cs_bill_cdemo_sk#98) AND isnotnull(cs_bill_customer_sk#97)) AND isnotnull(cs_item_sk#99)) + +(69) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#106, cd_dep_count#107] (70) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102] -Right output [4]: [c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Arguments: [cs_bill_customer_sk#92], [c_customer_sk#103], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] +Right output [2]: [cd_demo_sk#106, cd_dep_count#107] +Arguments: [cs_bill_cdemo_sk#98], [cd_demo_sk#106], Inner, BuildRight (71) CometProject -Input [13]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] +Input [11]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_demo_sk#106, cd_dep_count#107] +Arguments: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107], [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107] -(72) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#107] +(72) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] (73) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Right output [1]: [cd_demo_sk#107] -Arguments: [c_current_cdemo_sk#104], [cd_demo_sk#107], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107] +Right output [4]: [c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Arguments: [cs_bill_customer_sk#97], [c_customer_sk#108], Inner, BuildRight (74) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106, cd_demo_sk#107] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106] +Input [13]: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] -(75) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Arguments: [ca_address_sk#108, ca_state#109, ca_country#110] +(75) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#112] -(76) CometFilter -Input [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Condition : (ca_state#109 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#108)) +(76) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Right output [1]: [cd_demo_sk#112] +Arguments: [c_current_cdemo_sk#109], [cd_demo_sk#112], Inner, BuildRight (77) CometProject -Input [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Arguments: [ca_address_sk#108, ca_country#110], [ca_address_sk#108, ca_country#110] +Input [12]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111, cd_demo_sk#112] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111] -(78) CometBroadcastExchange -Input [2]: [ca_address_sk#108, ca_country#110] -Arguments: [ca_address_sk#108, ca_country#110] +(78) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(79) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106] -Right output [2]: [ca_address_sk#108, ca_country#110] -Arguments: [c_current_addr_sk#105], [ca_address_sk#108], Inner, BuildRight +(79) CometFilter +Input [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#114, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#113)) (80) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106, ca_address_sk#108, ca_country#110] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110] +Input [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Arguments: [ca_address_sk#113, ca_country#115], [ca_address_sk#113, ca_country#115] -(81) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#111] +(81) CometBroadcastExchange +Input [2]: [ca_address_sk#113, ca_country#115] +Arguments: [ca_address_sk#113, ca_country#115] (82) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110] -Right output [1]: [d_date_sk#111] -Arguments: [cs_sold_date_sk#100], [d_date_sk#111], Inner, BuildRight +Left output [10]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111] +Right output [2]: [ca_address_sk#113, ca_country#115] +Arguments: [c_current_addr_sk#110], [ca_address_sk#113], Inner, BuildRight (83) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110, d_date_sk#111] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110] +Input [12]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111, ca_address_sk#113, ca_country#115] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115] -(84) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#112, i_item_id#113] +(84) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#116] (85) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110] -Right output [2]: [i_item_sk#112, i_item_id#113] -Arguments: [cs_item_sk#94], [i_item_sk#112], Inner, BuildRight +Left output [10]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115] +Right output [1]: [d_date_sk#116] +Arguments: [cs_sold_date_sk#105], [d_date_sk#116], Inner, BuildRight (86) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110, i_item_sk#112, i_item_id#113] -Arguments: [i_item_id#113, ca_country#110, agg1#114, agg2#115, agg3#116, agg4#117, agg5#118, agg6#119, agg7#120], [i_item_id#113, ca_country#110, cast(cs_quantity#95 as decimal(12,2)) AS agg1#114, cast(cs_list_price#96 as decimal(12,2)) AS agg2#115, cast(cs_coupon_amt#98 as decimal(12,2)) AS agg3#116, cast(cs_sales_price#97 as decimal(12,2)) AS agg4#117, cast(cs_net_profit#99 as decimal(12,2)) AS agg5#118, cast(c_birth_year#106 as decimal(12,2)) AS agg6#119, cast(cd_dep_count#102 as decimal(12,2)) AS agg7#120] - -(87) CometHashAggregate -Input [9]: [i_item_id#113, ca_country#110, agg1#114, agg2#115, agg3#116, agg4#117, agg5#118, agg6#119, agg7#120] -Keys [2]: [i_item_id#113, ca_country#110] -Functions [7]: [partial_avg(agg1#114), partial_avg(agg2#115), partial_avg(agg3#116), partial_avg(agg4#117), partial_avg(agg5#118), partial_avg(agg6#119), partial_avg(agg7#120)] - -(88) CometExchange -Input [16]: [i_item_id#113, ca_country#110, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134] -Arguments: hashpartitioning(i_item_id#113, ca_country#110, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(89) CometHashAggregate -Input [16]: [i_item_id#113, ca_country#110, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134] -Keys [2]: [i_item_id#113, ca_country#110] -Functions [7]: [avg(agg1#114), avg(agg2#115), avg(agg3#116), avg(agg4#117), avg(agg5#118), avg(agg6#119), avg(agg7#120)] - -(90) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Arguments: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] - -(91) CometFilter -Input [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Condition : ((isnotnull(cs_bill_cdemo_sk#136) AND isnotnull(cs_bill_customer_sk#135)) AND isnotnull(cs_item_sk#137)) - -(92) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#144, cd_dep_count#145] - -(93) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Right output [2]: [cd_demo_sk#144, cd_dep_count#145] -Arguments: [cs_bill_cdemo_sk#136], [cd_demo_sk#144], Inner, BuildRight - -(94) CometProject -Input [11]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_demo_sk#144, cd_dep_count#145] -Arguments: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145], [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145] - -(95) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] +Input [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115, d_date_sk#116] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115] + +(87) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#117, i_item_id#29] + +(88) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115] +Right output [2]: [i_item_sk#117, i_item_id#29] +Arguments: [cs_item_sk#99], [i_item_sk#117], Inner, BuildRight + +(89) CometProject +Input [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115, i_item_sk#117, i_item_id#29] +Arguments: [i_item_id#29, ca_country#115, agg1#118, agg2#119, agg3#120, agg4#121, agg5#122, agg6#123, agg7#124], [i_item_id#29, ca_country#115, cast(cs_quantity#100 as decimal(12,2)) AS agg1#118, cast(cs_list_price#101 as decimal(12,2)) AS agg2#119, cast(cs_coupon_amt#103 as decimal(12,2)) AS agg3#120, cast(cs_sales_price#102 as decimal(12,2)) AS agg4#121, cast(cs_net_profit#104 as decimal(12,2)) AS agg5#122, cast(c_birth_year#111 as decimal(12,2)) AS agg6#123, cast(cd_dep_count#107 as decimal(12,2)) AS agg7#124] + +(90) CometHashAggregate +Input [9]: [i_item_id#29, ca_country#115, agg1#118, agg2#119, agg3#120, agg4#121, agg5#122, agg6#123, agg7#124] +Keys [2]: [i_item_id#29, ca_country#115] +Functions [7]: [partial_avg(agg1#118), partial_avg(agg2#119), partial_avg(agg3#120), partial_avg(agg4#121), partial_avg(agg5#122), partial_avg(agg6#123), partial_avg(agg7#124)] + +(91) CometExchange +Input [16]: [i_item_id#29, ca_country#115, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138] +Arguments: hashpartitioning(i_item_id#29, ca_country#115, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(92) CometHashAggregate +Input [16]: [i_item_id#29, ca_country#115, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138] +Keys [2]: [i_item_id#29, ca_country#115] +Functions [7]: [avg(agg1#118), avg(agg2#119), avg(agg3#120), avg(agg4#121), avg(agg5#122), avg(agg6#123), avg(agg7#124)] + +(93) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#147)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(94) CometFilter +Input [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] +Condition : ((isnotnull(cs_bill_cdemo_sk#140) AND isnotnull(cs_bill_customer_sk#139)) AND isnotnull(cs_item_sk#141)) + +(95) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#148, cd_dep_count#149] (96) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145] -Right output [4]: [c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Arguments: [cs_bill_customer_sk#135], [c_customer_sk#146], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] +Right output [2]: [cd_demo_sk#148, cd_dep_count#149] +Arguments: [cs_bill_cdemo_sk#140], [cd_demo_sk#148], Inner, BuildRight (97) CometProject -Input [13]: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] +Input [11]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_demo_sk#148, cd_dep_count#149] +Arguments: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149], [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149] -(98) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#150] +(98) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] (99) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Right output [1]: [cd_demo_sk#150] -Arguments: [c_current_cdemo_sk#147], [cd_demo_sk#150], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149] +Right output [4]: [c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Arguments: [cs_bill_customer_sk#139], [c_customer_sk#150], Inner, BuildRight (100) CometProject -Input [12]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149, cd_demo_sk#150] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149] +Input [13]: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] -(101) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#151, ca_state#152] -Arguments: [ca_address_sk#151, ca_state#152] +(101) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#154] -(102) CometFilter -Input [2]: [ca_address_sk#151, ca_state#152] -Condition : (ca_state#152 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#151)) +(102) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Right output [1]: [cd_demo_sk#154] +Arguments: [c_current_cdemo_sk#151], [cd_demo_sk#154], Inner, BuildRight (103) CometProject -Input [2]: [ca_address_sk#151, ca_state#152] -Arguments: [ca_address_sk#151], [ca_address_sk#151] +Input [12]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153, cd_demo_sk#154] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153] -(104) CometBroadcastExchange -Input [1]: [ca_address_sk#151] -Arguments: [ca_address_sk#151] +(104) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#155, ca_state#156] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(105) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149] -Right output [1]: [ca_address_sk#151] -Arguments: [c_current_addr_sk#148], [ca_address_sk#151], Inner, BuildRight +(105) CometFilter +Input [2]: [ca_address_sk#155, ca_state#156] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#156, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#155)) (106) CometProject -Input [11]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149, ca_address_sk#151] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149] +Input [2]: [ca_address_sk#155, ca_state#156] +Arguments: [ca_address_sk#155], [ca_address_sk#155] -(107) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#153] +(107) CometBroadcastExchange +Input [1]: [ca_address_sk#155] +Arguments: [ca_address_sk#155] (108) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149] -Right output [1]: [d_date_sk#153] -Arguments: [cs_sold_date_sk#143], [d_date_sk#153], Inner, BuildRight +Left output [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153] +Right output [1]: [ca_address_sk#155] +Arguments: [c_current_addr_sk#152], [ca_address_sk#155], Inner, BuildRight (109) CometProject -Input [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149, d_date_sk#153] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149] +Input [11]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153, ca_address_sk#155] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153] -(110) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#154, i_item_id#155] +(110) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#157] (111) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149] -Right output [2]: [i_item_sk#154, i_item_id#155] -Arguments: [cs_item_sk#137], [i_item_sk#154], Inner, BuildRight +Left output [9]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153] +Right output [1]: [d_date_sk#157] +Arguments: [cs_sold_date_sk#147], [d_date_sk#157], Inner, BuildRight (112) CometProject -Input [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149, i_item_sk#154, i_item_id#155] -Arguments: [i_item_id#155, agg1#156, agg2#157, agg3#158, agg4#159, agg5#160, agg6#161, agg7#162], [i_item_id#155, cast(cs_quantity#138 as decimal(12,2)) AS agg1#156, cast(cs_list_price#139 as decimal(12,2)) AS agg2#157, cast(cs_coupon_amt#141 as decimal(12,2)) AS agg3#158, cast(cs_sales_price#140 as decimal(12,2)) AS agg4#159, cast(cs_net_profit#142 as decimal(12,2)) AS agg5#160, cast(c_birth_year#149 as decimal(12,2)) AS agg6#161, cast(cd_dep_count#145 as decimal(12,2)) AS agg7#162] - -(113) CometHashAggregate -Input [8]: [i_item_id#155, agg1#156, agg2#157, agg3#158, agg4#159, agg5#160, agg6#161, agg7#162] -Keys [1]: [i_item_id#155] -Functions [7]: [partial_avg(agg1#156), partial_avg(agg2#157), partial_avg(agg3#158), partial_avg(agg4#159), partial_avg(agg5#160), partial_avg(agg6#161), partial_avg(agg7#162)] - -(114) CometExchange -Input [15]: [i_item_id#155, sum#163, count#164, sum#165, count#166, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176] -Arguments: hashpartitioning(i_item_id#155, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(115) CometHashAggregate -Input [15]: [i_item_id#155, sum#163, count#164, sum#165, count#166, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176] -Keys [1]: [i_item_id#155] -Functions [7]: [avg(agg1#156), avg(agg2#157), avg(agg3#158), avg(agg4#159), avg(agg5#160), avg(agg6#161), avg(agg7#162)] - -(116) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Arguments: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] - -(117) CometFilter -Input [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Condition : ((isnotnull(cs_bill_cdemo_sk#178) AND isnotnull(cs_bill_customer_sk#177)) AND isnotnull(cs_item_sk#179)) - -(118) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#186, cd_dep_count#187] - -(119) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Right output [2]: [cd_demo_sk#186, cd_dep_count#187] -Arguments: [cs_bill_cdemo_sk#178], [cd_demo_sk#186], Inner, BuildRight - -(120) CometProject -Input [11]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_demo_sk#186, cd_dep_count#187] -Arguments: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187], [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187] - -(121) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] +Input [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153, d_date_sk#157] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153] + +(113) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#158, i_item_id#29] + +(114) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153] +Right output [2]: [i_item_sk#158, i_item_id#29] +Arguments: [cs_item_sk#141], [i_item_sk#158], Inner, BuildRight + +(115) CometProject +Input [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153, i_item_sk#158, i_item_id#29] +Arguments: [i_item_id#29, agg1#159, agg2#160, agg3#161, agg4#162, agg5#163, agg6#164, agg7#165], [i_item_id#29, cast(cs_quantity#142 as decimal(12,2)) AS agg1#159, cast(cs_list_price#143 as decimal(12,2)) AS agg2#160, cast(cs_coupon_amt#145 as decimal(12,2)) AS agg3#161, cast(cs_sales_price#144 as decimal(12,2)) AS agg4#162, cast(cs_net_profit#146 as decimal(12,2)) AS agg5#163, cast(c_birth_year#153 as decimal(12,2)) AS agg6#164, cast(cd_dep_count#149 as decimal(12,2)) AS agg7#165] + +(116) CometHashAggregate +Input [8]: [i_item_id#29, agg1#159, agg2#160, agg3#161, agg4#162, agg5#163, agg6#164, agg7#165] +Keys [1]: [i_item_id#29] +Functions [7]: [partial_avg(agg1#159), partial_avg(agg2#160), partial_avg(agg3#161), partial_avg(agg4#162), partial_avg(agg5#163), partial_avg(agg6#164), partial_avg(agg7#165)] + +(117) CometExchange +Input [15]: [i_item_id#29, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(118) CometHashAggregate +Input [15]: [i_item_id#29, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179] +Keys [1]: [i_item_id#29] +Functions [7]: [avg(agg1#159), avg(agg2#160), avg(agg3#161), avg(agg4#162), avg(agg5#163), avg(agg6#164), avg(agg7#165)] + +(119) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#188)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(120) CometFilter +Input [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] +Condition : ((isnotnull(cs_bill_cdemo_sk#181) AND isnotnull(cs_bill_customer_sk#180)) AND isnotnull(cs_item_sk#182)) + +(121) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#189, cd_dep_count#190] (122) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187] -Right output [4]: [c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Arguments: [cs_bill_customer_sk#177], [c_customer_sk#188], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] +Right output [2]: [cd_demo_sk#189, cd_dep_count#190] +Arguments: [cs_bill_cdemo_sk#181], [cd_demo_sk#189], Inner, BuildRight (123) CometProject -Input [13]: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] +Input [11]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_demo_sk#189, cd_dep_count#190] +Arguments: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190], [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190] -(124) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#192] +(124) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] (125) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Right output [1]: [cd_demo_sk#192] -Arguments: [c_current_cdemo_sk#189], [cd_demo_sk#192], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190] +Right output [4]: [c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Arguments: [cs_bill_customer_sk#180], [c_customer_sk#191], Inner, BuildRight (126) CometProject -Input [12]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191, cd_demo_sk#192] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191] +Input [13]: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] -(127) ReusedExchange [Reuses operator id: 104] -Output [1]: [ca_address_sk#193] +(127) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#195] (128) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191] -Right output [1]: [ca_address_sk#193] -Arguments: [c_current_addr_sk#190], [ca_address_sk#193], Inner, BuildRight +Left output [11]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Right output [1]: [cd_demo_sk#195] +Arguments: [c_current_cdemo_sk#192], [cd_demo_sk#195], Inner, BuildRight (129) CometProject -Input [11]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191, ca_address_sk#193] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191] +Input [12]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194, cd_demo_sk#195] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194] -(130) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#194] +(130) ReusedExchange [Reuses operator id: 107] +Output [1]: [ca_address_sk#196] (131) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191] -Right output [1]: [d_date_sk#194] -Arguments: [cs_sold_date_sk#185], [d_date_sk#194], Inner, BuildRight +Left output [10]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194] +Right output [1]: [ca_address_sk#196] +Arguments: [c_current_addr_sk#193], [ca_address_sk#196], Inner, BuildRight (132) CometProject -Input [10]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191, d_date_sk#194] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191] +Input [11]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194, ca_address_sk#196] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194] -(133) ReusedExchange [Reuses operator id: 17] -Output [1]: [i_item_sk#195] +(133) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#197] (134) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191] -Right output [1]: [i_item_sk#195] -Arguments: [cs_item_sk#179], [i_item_sk#195], Inner, BuildRight +Left output [9]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194] +Right output [1]: [d_date_sk#197] +Arguments: [cs_sold_date_sk#188], [d_date_sk#197], Inner, BuildRight (135) CometProject -Input [9]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191, i_item_sk#195] -Arguments: [agg1#196, agg2#197, agg3#198, agg4#199, agg5#200, agg6#201, agg7#202], [cast(cs_quantity#180 as decimal(12,2)) AS agg1#196, cast(cs_list_price#181 as decimal(12,2)) AS agg2#197, cast(cs_coupon_amt#183 as decimal(12,2)) AS agg3#198, cast(cs_sales_price#182 as decimal(12,2)) AS agg4#199, cast(cs_net_profit#184 as decimal(12,2)) AS agg5#200, cast(c_birth_year#191 as decimal(12,2)) AS agg6#201, cast(cd_dep_count#187 as decimal(12,2)) AS agg7#202] - -(136) CometHashAggregate -Input [7]: [agg1#196, agg2#197, agg3#198, agg4#199, agg5#200, agg6#201, agg7#202] +Input [10]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194, d_date_sk#197] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194] + +(136) CometNativeScan parquet spark_catalog.default.item +Output [1]: [i_item_sk#198] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(137) CometFilter +Input [1]: [i_item_sk#198] +Condition : isnotnull(i_item_sk#198) + +(138) CometBroadcastExchange +Input [1]: [i_item_sk#198] +Arguments: [i_item_sk#198] + +(139) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194] +Right output [1]: [i_item_sk#198] +Arguments: [cs_item_sk#182], [i_item_sk#198], Inner, BuildRight + +(140) CometProject +Input [9]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194, i_item_sk#198] +Arguments: [agg1#199, agg2#200, agg3#201, agg4#202, agg5#203, agg6#204, agg7#205], [cast(cs_quantity#183 as decimal(12,2)) AS agg1#199, cast(cs_list_price#184 as decimal(12,2)) AS agg2#200, cast(cs_coupon_amt#186 as decimal(12,2)) AS agg3#201, cast(cs_sales_price#185 as decimal(12,2)) AS agg4#202, cast(cs_net_profit#187 as decimal(12,2)) AS agg5#203, cast(c_birth_year#194 as decimal(12,2)) AS agg6#204, cast(cd_dep_count#190 as decimal(12,2)) AS agg7#205] + +(141) CometHashAggregate +Input [7]: [agg1#199, agg2#200, agg3#201, agg4#202, agg5#203, agg6#204, agg7#205] Keys: [] -Functions [7]: [partial_avg(agg1#196), partial_avg(agg2#197), partial_avg(agg3#198), partial_avg(agg4#199), partial_avg(agg5#200), partial_avg(agg6#201), partial_avg(agg7#202)] +Functions [7]: [partial_avg(agg1#199), partial_avg(agg2#200), partial_avg(agg3#201), partial_avg(agg4#202), partial_avg(agg5#203), partial_avg(agg6#204), partial_avg(agg7#205)] -(137) CometExchange -Input [14]: [sum#203, count#204, sum#205, count#206, sum#207, count#208, sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216] +(142) CometExchange +Input [14]: [sum#206, count#207, sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(138) CometHashAggregate -Input [14]: [sum#203, count#204, sum#205, count#206, sum#207, count#208, sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216] +(143) CometHashAggregate +Input [14]: [sum#206, count#207, sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219] Keys: [] -Functions [7]: [avg(agg1#196), avg(agg2#197), avg(agg3#198), avg(agg4#199), avg(agg5#200), avg(agg6#201), avg(agg7#202)] +Functions [7]: [avg(agg1#199), avg(agg2#200), avg(agg3#201), avg(agg4#202), avg(agg5#203), avg(agg6#204), avg(agg7#205)] -(139) CometUnion -Child 0 Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] -Child 1 Input [11]: [i_item_id#70, ca_country#67, ca_state#66, county#224, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231] -Child 2 Input [11]: [i_item_id#113, ca_country#110, ca_state#232, county#233, agg1#234, agg2#235, agg3#236, agg4#237, agg5#238, agg6#239, agg7#240] -Child 3 Input [11]: [i_item_id#155, ca_country#241, ca_state#242, county#243, agg1#244, agg2#245, agg3#246, agg4#247, agg5#248, agg6#249, agg7#250] -Child 4 Input [11]: [i_item_id#251, ca_country#252, ca_state#253, county#254, agg1#255, agg2#256, agg3#257, agg4#258, agg5#259, agg6#260, agg7#261] +(144) CometUnion +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] +Child 1 Input [11]: [i_item_id#29, ca_country#73, ca_state#24, county#227, agg1#228, agg2#229, agg3#230, agg4#231, agg5#232, agg6#233, agg7#234] +Child 2 Input [11]: [i_item_id#29, ca_country#115, ca_state#235, county#236, agg1#237, agg2#238, agg3#239, agg4#240, agg5#241, agg6#242, agg7#243] +Child 3 Input [11]: [i_item_id#29, ca_country#244, ca_state#245, county#246, agg1#247, agg2#248, agg3#249, agg4#250, agg5#251, agg6#252, agg7#253] +Child 4 Input [11]: [i_item_id#254, ca_country#255, ca_state#256, county#257, agg1#258, agg2#259, agg3#260, agg4#261, agg5#262, agg6#263, agg7#264] -(140) CometTakeOrderedAndProject -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#23 ASC NULLS FIRST,ca_state#22 ASC NULLS FIRST,ca_county#21 ASC NULLS FIRST,i_item_id#27 ASC NULLS FIRST], output=[i_item_id#27,ca_country#23,ca_state#22,ca_county#21,agg1#217,agg2#218,agg3#219,agg4#220,agg5#221,agg6#222,agg7#223]), [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223], 100, [ca_country#23 ASC NULLS FIRST, ca_state#22 ASC NULLS FIRST, ca_county#21 ASC NULLS FIRST, i_item_id#27 ASC NULLS FIRST], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] +(145) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#220,agg2#221,agg3#222,agg4#223,agg5#224,agg6#225,agg7#226]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] -(141) ColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] +(146) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/extended.txt new file mode 100644 index 0000000000..59e46c3c8e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/extended.txt @@ -0,0 +1,204 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 201 out of 201 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/simplified.txt index e4f78115a8..a5655241a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] @@ -19,31 +19,33 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state,ca_country] + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id,ca_country,ca_state] #8 - CometHashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] @@ -57,18 +59,19 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] + CometProject [ca_state] [ca_address_sk,ca_state,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] ReusedExchange [d_date_sk] #6 ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id,ca_country] #10 - CometHashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] @@ -82,19 +85,19 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 CometBroadcastExchange [ca_address_sk,ca_country] #11 CometProject [ca_address_sk,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] ReusedExchange [d_date_sk] #6 ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id] #12 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] @@ -108,19 +111,19 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 CometBroadcastExchange [ca_address_sk] #13 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] ReusedExchange [d_date_sk] #6 ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange #14 - CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] @@ -134,10 +137,12 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 ReusedExchange [ca_address_sk] #13 ReusedExchange [d_date_sk] #6 - ReusedExchange [i_item_sk] #4 + CometBroadcastExchange [i_item_sk] #15 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/explain.txt index 48efd7d6ca..23634911fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/explain.txt @@ -1,116 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/extended.txt new file mode 100644 index 0000000000..7fd40cf7dd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/simplified.txt index 5fc5ba1ef8..06e56fab81 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] CometExchange [i_class] #1 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/explain.txt index eefb6710c5..6c7ac14107 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/explain.txt @@ -1,41 +1,50 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * Expand (20) - +- * Project (19) - +- * BroadcastNestedLoopJoin Inner BuildRight (18) - :- * ColumnarToRow (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- BroadcastExchange (17) - +- * ColumnarToRow (16) - +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` +TakeOrderedAndProject (26) ++- * HashAggregate (25) + +- * CometColumnarToRow (24) + +- CometColumnarExchange (23) + +- * HashAggregate (22) + +- * Expand (21) + +- * Project (20) + +- * BroadcastNestedLoopJoin Inner BuildRight (19) + :- * CometColumnarToRow (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- BroadcastExchange (18) + +- * CometColumnarToRow (17) + +- CometNativeScan parquet spark_catalog.default.warehouse (16) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#3)] +PushedFilters: [IsNotNull(inv_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -58,71 +67,84 @@ Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Condition : isnotnull(i_item_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#7, 50, true, false, true) AS i_brand#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#10, 50, true, false, true) AS i_product_name#14] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(13) CometBroadcastHashJoin Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] -Right output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight -(13) CometProject -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10], [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(14) CometProject +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(14) ColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(15) CometColumnarToRow [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(15) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(16) CometNativeScan parquet spark_catalog.default.warehouse Output: [] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +ReadSchema: struct<> -(16) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input: [] -(17) BroadcastExchange +(18) BroadcastExchange Input: [] Arguments: IdentityBroadcastMode, [plan_id=1] -(18) BroadcastNestedLoopJoin [codegen id : 2] +(19) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(19) Project [codegen id : 2] -Output [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(20) Project [codegen id : 2] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(20) Expand [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Arguments: [[inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9, 0], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, null, 1], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, null, null, 3], [inv_quantity_on_hand#2, i_product_name#10, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(21) Expand [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] -(21) HashAggregate [codegen id : 2] -Input [6]: [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(22) HashAggregate [codegen id : 2] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#16, count#17] -Results [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(23) CometColumnarExchange +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(22) Exchange -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(24) CometColumnarToRow [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] -(23) HashAggregate [codegen id : 3] -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(25) HashAggregate [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#20] -Results [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, avg(inv_quantity_on_hand#2)#20 AS qoh#21] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] -(24) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] -Arguments: 100, [qoh#21 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#12 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_category#14 ASC NULLS FIRST], [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] +(26) TakeOrderedAndProject +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/extended.txt new file mode 100644 index 0000000000..e4737a8934 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/extended.txt @@ -0,0 +1,28 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Project + +- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 16 out of 23 eligible operators (69%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/simplified.txt index febd5d2e62..199ad41e45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/simplified.txt @@ -1,31 +1,33 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] WholeStageCodegen (3) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometNativeScan: `spark_catalog`.`default`.`warehouse` + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometNativeScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/explain.txt index 7eeb898493..47d33e39f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/explain.txt @@ -1,63 +1,71 @@ == Physical Plan == -* ColumnarToRow (46) -+- CometTakeOrderedAndProject (45) - +- CometUnion (44) - :- CometHashAggregate (23) - : +- CometHashAggregate (22) - : +- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometUnion (45) + :- CometHashAggregate (24) + : +- CometHashAggregate (23) + : +- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (14) - :- CometHashAggregate (28) - : +- CometExchange (27) - : +- CometHashAggregate (26) - : +- CometHashAggregate (25) - : +- ReusedExchange (24) - :- CometHashAggregate (33) - : +- CometExchange (32) - : +- CometHashAggregate (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - :- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometHashAggregate (35) - : +- ReusedExchange (34) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometHashAggregate (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.warehouse (15) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometHashAggregate (26) + : +- ReusedExchange (25) + :- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + :- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometHashAggregate (36) + : +- ReusedExchange (35) + +- CometHashAggregate (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometHashAggregate (41) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] @@ -80,171 +88,181 @@ Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Condition : isnotnull(i_item_sk#7) -(11) CometBroadcastExchange +(11) CometProject Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(13) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(15) CometNativeScan parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) + +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] + +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight + +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(20) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(20) CometExchange -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#13, count#14] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#13, count#14] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] +(21) CometExchange +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#15] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(qoh#15)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#3)] (23) CometHashAggregate -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#16, count#17] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(qoh#15)] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#23] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [partial_avg(qoh#23)] -(24) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, sum#22, count#23] +(24) CometHashAggregate +Input [6]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, sum#24, count#25] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [avg(qoh#23)] -(25) CometHashAggregate -Input [6]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, sum#22, count#23] -Keys [4]: [i_product_name#18, i_brand#19, i_class#20, i_category#21] -Functions [1]: [avg(inv_quantity_on_hand#24)] +(25) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] (26) CometHashAggregate -Input [4]: [i_product_name#18, i_brand#19, i_class#20, qoh#25] -Keys [3]: [i_product_name#18, i_brand#19, i_class#20] -Functions [1]: [partial_avg(qoh#25)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#28)] -(27) CometExchange -Input [5]: [i_product_name#18, i_brand#19, i_class#20, sum#26, count#27] -Arguments: hashpartitioning(i_product_name#18, i_brand#19, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, i_class#13, qoh#29] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [partial_avg(qoh#29)] -(28) CometHashAggregate -Input [5]: [i_product_name#18, i_brand#19, i_class#20, sum#26, count#27] -Keys [3]: [i_product_name#18, i_brand#19, i_class#20] -Functions [1]: [avg(qoh#25)] +(28) CometExchange +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#30, count#31] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(29) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#28, i_brand#29, i_class#30, i_category#31, sum#32, count#33] +(29) CometHashAggregate +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#30, count#31] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [avg(qoh#29)] -(30) CometHashAggregate -Input [6]: [i_product_name#28, i_brand#29, i_class#30, i_category#31, sum#32, count#33] -Keys [4]: [i_product_name#28, i_brand#29, i_class#30, i_category#31] -Functions [1]: [avg(inv_quantity_on_hand#34)] +(30) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#32, count#33] (31) CometHashAggregate -Input [3]: [i_product_name#28, i_brand#29, qoh#35] -Keys [2]: [i_product_name#28, i_brand#29] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#32, count#33] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#34)] + +(32) CometHashAggregate +Input [3]: [i_product_name#15, i_brand#12, qoh#35] +Keys [2]: [i_product_name#15, i_brand#12] Functions [1]: [partial_avg(qoh#35)] -(32) CometExchange -Input [4]: [i_product_name#28, i_brand#29, sum#36, count#37] -Arguments: hashpartitioning(i_product_name#28, i_brand#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(33) CometExchange +Input [4]: [i_product_name#15, i_brand#12, sum#36, count#37] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(33) CometHashAggregate -Input [4]: [i_product_name#28, i_brand#29, sum#36, count#37] -Keys [2]: [i_product_name#28, i_brand#29] +(34) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, sum#36, count#37] +Keys [2]: [i_product_name#15, i_brand#12] Functions [1]: [avg(qoh#35)] -(34) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#38, i_brand#39, i_class#40, i_category#41, sum#42, count#43] - -(35) CometHashAggregate -Input [6]: [i_product_name#38, i_brand#39, i_class#40, i_category#41, sum#42, count#43] -Keys [4]: [i_product_name#38, i_brand#39, i_class#40, i_category#41] -Functions [1]: [avg(inv_quantity_on_hand#44)] +(35) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#38, count#39] (36) CometHashAggregate -Input [2]: [i_product_name#38, qoh#45] -Keys [1]: [i_product_name#38] -Functions [1]: [partial_avg(qoh#45)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#38, count#39] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#40)] -(37) CometExchange -Input [3]: [i_product_name#38, sum#46, count#47] -Arguments: hashpartitioning(i_product_name#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(37) CometHashAggregate +Input [2]: [i_product_name#15, qoh#41] +Keys [1]: [i_product_name#15] +Functions [1]: [partial_avg(qoh#41)] -(38) CometHashAggregate -Input [3]: [i_product_name#38, sum#46, count#47] -Keys [1]: [i_product_name#38] -Functions [1]: [avg(qoh#45)] +(38) CometExchange +Input [3]: [i_product_name#15, sum#42, count#43] +Arguments: hashpartitioning(i_product_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(39) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#48, i_brand#49, i_class#50, i_category#51, sum#52, count#53] +(39) CometHashAggregate +Input [3]: [i_product_name#15, sum#42, count#43] +Keys [1]: [i_product_name#15] +Functions [1]: [avg(qoh#41)] -(40) CometHashAggregate -Input [6]: [i_product_name#48, i_brand#49, i_class#50, i_category#51, sum#52, count#53] -Keys [4]: [i_product_name#48, i_brand#49, i_class#50, i_category#51] -Functions [1]: [avg(inv_quantity_on_hand#54)] +(40) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#44, count#45] (41) CometHashAggregate -Input [1]: [qoh#55] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#44, count#45] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#46)] + +(42) CometHashAggregate +Input [1]: [qoh#47] Keys: [] -Functions [1]: [partial_avg(qoh#55)] +Functions [1]: [partial_avg(qoh#47)] -(42) CometExchange -Input [2]: [sum#56, count#57] +(43) CometExchange +Input [2]: [sum#48, count#49] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(43) CometHashAggregate -Input [2]: [sum#56, count#57] +(44) CometHashAggregate +Input [2]: [sum#48, count#49] Keys: [] -Functions [1]: [avg(qoh#55)] +Functions [1]: [avg(qoh#47)] -(44) CometUnion -Child 0 Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] -Child 1 Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#59, qoh#60] -Child 2 Input [5]: [i_product_name#28, i_brand#29, i_class#61, i_category#62, qoh#63] -Child 3 Input [5]: [i_product_name#38, i_brand#64, i_class#65, i_category#66, qoh#67] -Child 4 Input [5]: [i_product_name#68, i_brand#69, i_class#70, i_category#71, qoh#72] +(45) CometUnion +Child 0 Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] +Child 1 Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#51, qoh#52] +Child 2 Input [5]: [i_product_name#15, i_brand#12, i_class#53, i_category#54, qoh#55] +Child 3 Input [5]: [i_product_name#15, i_brand#56, i_class#57, i_category#58, qoh#59] +Child 4 Input [5]: [i_product_name#60, i_brand#61, i_class#62, i_category#63, qoh#64] -(45) CometTakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#58 ASC NULLS FIRST,i_product_name#11 ASC NULLS FIRST,i_brand#8 ASC NULLS FIRST,i_class#9 ASC NULLS FIRST,i_category#10 ASC NULLS FIRST], output=[i_product_name#11,i_brand#8,i_class#9,i_category#10,qoh#58]), [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58], 100, [qoh#58 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#8 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] +(46) CometTakeOrderedAndProject +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#50 ASC NULLS FIRST,i_product_name#19 ASC NULLS FIRST,i_brand#20 ASC NULLS FIRST,i_class#21 ASC NULLS FIRST,i_category#22 ASC NULLS FIRST], output=[i_product_name#19,i_brand#20,i_class#21,i_category#22,qoh#50]), [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50], 100, 0, [qoh#50 ASC NULLS FIRST, i_product_name#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_class#21 ASC NULLS FIRST, i_category#22 ASC NULLS FIRST], [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] -(46) ColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] +(47) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/extended.txt new file mode 100644 index 0000000000..ed898ac4d5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/extended.txt @@ -0,0 +1,129 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 126 out of 126 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/simplified.txt index 43704500c6..d404363103 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/simplified.txt @@ -1,13 +1,13 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] CometUnion [i_product_name,i_brand,i_class,i_category,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,i_category,sum,count] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,i_product_name,i_brand,i_class,i_category,avg(inv_quantity_on_hand)] CometExchange [i_product_name,i_brand,i_class,i_category] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,sum,count] CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] @@ -15,34 +15,35 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange [i_product_name,i_brand,i_class] #5 - CometHashAggregate [i_product_name,i_brand,i_class,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,qoh,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] + CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange [i_product_name,i_brand] #6 - CometHashAggregate [i_product_name,i_brand,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,qoh,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] + CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange [i_product_name] #7 - CometHashAggregate [i_product_name,sum,count,qoh] - CometHashAggregate [i_product_name,qoh,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [i_product_name,sum,count] + CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange #8 - CometHashAggregate [sum,count,qoh] - CometHashAggregate [qoh,i_product_name,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [sum,count] + CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/explain.txt index 8b77e3c896..696a4579e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/explain.txt @@ -1,54 +1,62 @@ == Physical Plan == -* ColumnarToRow (45) -+- CometSort (44) - +- CometColumnarExchange (43) - +- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- * Filter (47) + +- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] @@ -66,9 +74,12 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] @@ -95,302 +106,348 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometNativeScan: `spark_catalog`.`default`.`store` +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(25) CometFilter -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) - -(26) CometBroadcastExchange -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(29) ColumnarToRow [codegen id : 2] -Input [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(31) CometFilter -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) - -(32) ColumnarToRow [codegen id : 1] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(33) BroadcastExchange -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Condition : ((isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) AND isnotnull(c_birth_country#30)) + +(27) CometProject +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [c_customer_sk#26, c_current_addr_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#32, c_birth_country#30] + +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(31) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Condition : ((isnotnull(ca_address_sk#33) AND isnotnull(ca_country#36)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true))) + +(34) CometProject +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Arguments: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36], [ca_address_sk#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#34, 2, true, false, true) AS ca_state#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true) AS ca_zip#38, ca_country#36] + +(35) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] + +(36) BroadcastExchange +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] +Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[3, string, true]), input[2, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#27, c_birth_country#30, s_zip#16] +Right keys [3]: [ca_address_sk#33, upper(ca_country#36), ca_zip#38] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30, ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#30] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] +Aggregate Attributes [1]: [sum#39] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] -(37) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [partial_sum(netpaid#33)] -Aggregate Attributes [2]: [sum#34, isEmpty#35] -Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(40) Exchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [sum(netpaid#33)] -Aggregate Attributes [1]: [sum(netpaid#33)#38] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) - -(43) CometColumnarExchange -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(44) CometSort -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] - -(45) ColumnarToRow [codegen id : 5] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] - -===== Subqueries ===== +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (72) -+- Exchange (71) - +- * HashAggregate (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * ColumnarToRow (63) - : +- CometProject (62) - : +- CometBroadcastHashJoin (61) - : :- CometProject (59) - : : +- CometBroadcastHashJoin (58) - : : :- CometProject (54) - : : : +- CometBroadcastHashJoin (53) - : : : :- CometProject (51) - : : : : +- CometSortMergeJoin (50) - : : : : :- CometSort (47) - : : : : : +- ReusedExchange (46) - : : : : +- CometSort (49) - : : : : +- ReusedExchange (48) - : : : +- ReusedExchange (52) - : : +- CometBroadcastExchange (57) - : : +- CometFilter (56) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (55) - : +- ReusedExchange (60) - +- ReusedExchange (64) - - -(46) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] - -(47) CometSort -Input [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46], [ss_ticket_number#45 ASC NULLS FIRST, ss_item_sk#42 ASC NULLS FIRST] - -(48) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#47, sr_ticket_number#48] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#41] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#41,17,2) AS netpaid#42] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, netpaid#42] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [partial_sum(netpaid#42)] +Aggregate Attributes [2]: [sum#43, isEmpty#44] +Results [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(44) CometColumnarExchange +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [sum(netpaid#42)] +Aggregate Attributes [1]: [sum(netpaid#42)#47] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, sum(netpaid#42)#47 AS paid#48] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Condition : (isnotnull(paid#48) AND (cast(paid#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8)))) + +(48) CometColumnarExchange +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: rangepartitioning(c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (49) CometSort -Input [2]: [sr_item_sk#47, sr_ticket_number#48] -Arguments: [sr_item_sk#47, sr_ticket_number#48], [sr_ticket_number#48 ASC NULLS FIRST, sr_item_sk#47 ASC NULLS FIRST] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48], [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] -(50) CometSortMergeJoin -Left output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -Right output [2]: [sr_item_sk#47, sr_ticket_number#48] -Arguments: [ss_ticket_number#45, ss_item_sk#42], [sr_ticket_number#48, sr_item_sk#47], Inner +(50) CometColumnarToRow [codegen id : 5] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] -(51) CometProject -Input [7]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46, sr_item_sk#47, sr_ticket_number#48] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46], [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] +===== Subqueries ===== -(52) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* HashAggregate (80) ++- * CometColumnarToRow (79) + +- CometColumnarExchange (78) + +- * HashAggregate (77) + +- * HashAggregate (76) + +- * CometColumnarToRow (75) + +- CometColumnarExchange (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- * CometColumnarToRow (69) + : +- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (56) + : : : : +- CometSortMergeJoin (55) + : : : : :- CometSort (52) + : : : : : +- ReusedExchange (51) + : : : : +- CometSort (54) + : : : : +- ReusedExchange (53) + : : : +- ReusedExchange (57) + : : +- CometBroadcastExchange (63) + : : +- CometProject (62) + : : +- CometFilter (61) + : : +- CometNativeScan parquet spark_catalog.default.item (60) + : +- ReusedExchange (66) + +- ReusedExchange (70) + + +(51) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] + +(52) CometSort +Input [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55], [ss_ticket_number#54 ASC NULLS FIRST, ss_item_sk#51 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#56, sr_ticket_number#57] + +(54) CometSort +Input [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [sr_item_sk#56, sr_ticket_number#57], [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Right output [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_ticket_number#54, ss_item_sk#51], [sr_ticket_number#57, sr_item_sk#56], Inner + +(56) CometProject +Input [7]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55, sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55], [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] + +(57) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] -(53) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] -Right output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -Arguments: [ss_store_sk#44], [s_store_sk#49], Inner, BuildRight +(58) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] +Right output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_store_sk#53], [s_store_sk#58], Inner, BuildRight -(54) CometProject -Input [8]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46, s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52], [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] +(59) CometProject +Input [8]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55, s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16], [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] -(55) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] +(60) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(56) CometFilter -Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Condition : isnotnull(i_item_sk#53) +(61) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) -(57) CometBroadcastExchange -Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] +(62) CometProject +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [i_item_sk#60, i_current_price#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#62, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#63, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#64, 10, true, false, true) AS i_units#25, i_manager_id#65] -(58) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] -Right output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [ss_item_sk#42], [i_item_sk#53], Inner, BuildRight +(63) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(59) CometProject -Input [12]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58], [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] +(64) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_item_sk#51], [i_item_sk#60], Inner, BuildRight -(60) ReusedExchange [Reuses operator id: 26] -Output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] +(65) CometProject +Input [12]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(61) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Right output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -Arguments: [ss_customer_sk#43], [c_customer_sk#59], Inner, BuildRight +(66) ReusedExchange [Reuses operator id: 28] +Output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(62) CometProject -Input [15]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -Arguments: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63], [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] +(67) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Right output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_customer_sk#52], [c_customer_sk#66], Inner, BuildRight + +(68) CometProject +Input [15]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68], [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(63) ColumnarToRow [codegen id : 2] -Input [13]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] +(69) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(64) ReusedExchange [Reuses operator id: 33] -Output [4]: [ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] +(70) ReusedExchange [Reuses operator id: 36] +Output [4]: [ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] -(65) BroadcastHashJoin [codegen id : 2] -Left keys [3]: [c_current_addr_sk#60, c_birth_country#63, s_zip#52] -Right keys [3]: [ca_address_sk#64, upper(ca_country#67), ca_zip#66] +(71) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#67, c_birth_country#68, s_zip#16] +Right keys [3]: [ca_address_sk#69, upper(ca_country#70), ca_zip#38] Join type: Inner Join condition: None -(66) Project [codegen id : 2] -Output [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] -Input [17]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63, ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] - -(67) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] -Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#46))] -Aggregate Attributes [1]: [sum#68] -Results [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] - -(68) Exchange -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Arguments: hashpartitioning(c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(69) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] -Functions [1]: [sum(UnscaledValue(ss_net_paid#46))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#46))#32] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#46))#32,17,2) AS netpaid#70] - -(70) HashAggregate [codegen id : 3] -Input [1]: [netpaid#70] +(72) Project [codegen id : 2] +Output [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68, ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] + +(73) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum#71] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(74) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(75) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(76) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#55))#41] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#55))#41,17,2) AS netpaid#73] + +(77) HashAggregate [codegen id : 3] +Input [1]: [netpaid#73] Keys: [] -Functions [1]: [partial_avg(netpaid#70)] -Aggregate Attributes [2]: [sum#71, count#72] -Results [2]: [sum#73, count#74] +Functions [1]: [partial_avg(netpaid#73)] +Aggregate Attributes [2]: [sum#74, count#75] +Results [2]: [sum#76, count#77] + +(78) CometColumnarExchange +Input [2]: [sum#76, count#77] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] -(71) Exchange -Input [2]: [sum#73, count#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(79) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#76, count#77] -(72) HashAggregate [codegen id : 4] -Input [2]: [sum#73, count#74] +(80) HashAggregate [codegen id : 4] +Input [2]: [sum#76, count#77] Keys: [] -Functions [1]: [avg(netpaid#70)] -Aggregate Attributes [1]: [avg(netpaid#70)#75] -Results [1]: [(0.05 * avg(netpaid#70)#75) AS (0.05 * avg(netpaid))#76] +Functions [1]: [avg(netpaid#73)] +Aggregate Attributes [1]: [avg(netpaid#73)#78] +Results [1]: [(0.05 * avg(netpaid#73)#78) AS (0.05 * avg(netpaid))#79] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/extended.txt new file mode 100644 index 0000000000..13317e59ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/extended.txt @@ -0,0 +1,99 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Filter + : +- Subquery + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 72 out of 88 eligible operators (81%). Final plan contains 9 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/simplified.txt index c52c1b407a..b24e51723a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [c_last_name,c_first_name,s_store_name,paid] CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 @@ -8,18 +8,56 @@ WholeStageCodegen (5) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #10 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] @@ -30,62 +68,32 @@ WholeStageCodegen (5) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #4 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #5 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_zip,ca_country] + BroadcastExchange #9 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/explain.txt index 8b866beb2d..0aa5dc8238 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/explain.txt @@ -1,90 +1,99 @@ == Physical Plan == -* ColumnarToRow (69) -+- CometTakeOrderedAndProject (68) - +- CometUnion (67) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometUnion (69) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) : : :- CometProject (14) : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - :- CometHashAggregate (47) - : +- CometExchange (46) - : +- CometHashAggregate (45) - : +- CometProject (44) - : +- CometBroadcastHashJoin (43) - : :- CometProject (41) - : : +- CometBroadcastHashJoin (40) - : : :- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometProject (32) - : : : : +- CometBroadcastHashJoin (31) - : : : : :- CometFilter (29) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (28) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (39) - : : +- CometProject (38) - : : +- CometFilter (37) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (36) - : +- ReusedExchange (42) - +- CometHashAggregate (66) - +- CometExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometBroadcastHashJoin (62) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometFilter (49) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (48) - : : : +- ReusedExchange (50) - : : +- ReusedExchange (53) - : +- ReusedExchange (56) - +- CometBroadcastExchange (61) - +- CometFilter (60) - +- CometNativeScan: `spark_catalog`.`default`.`item` (59) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.store (15) + : +- CometBroadcastExchange (24) + : +- CometProject (23) + : +- CometFilter (22) + : +- CometNativeScan parquet spark_catalog.default.item (21) + :- CometHashAggregate (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (37) + : : : +- CometBroadcastHashJoin (36) + : : : :- CometProject (34) + : : : : +- CometBroadcastHashJoin (33) + : : : : :- CometFilter (31) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (30) + : : : : +- ReusedExchange (32) + : : : +- ReusedExchange (35) + : : +- CometBroadcastExchange (41) + : : +- CometProject (40) + : : +- CometFilter (39) + : : +- CometNativeScan parquet spark_catalog.default.store (38) + : +- ReusedExchange (44) + +- CometHashAggregate (68) + +- CometExchange (67) + +- CometHashAggregate (66) + +- CometProject (65) + +- CometBroadcastHashJoin (64) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- ReusedExchange (58) + +- CometBroadcastExchange (63) + +- CometFilter (62) + +- CometNativeScan parquet spark_catalog.default.item (61) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = F)) AND (cd_marital_status#11 = W)) AND (cd_education_status#12 = Primary )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -103,9 +112,12 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#13, d_year#14] @@ -128,233 +140,261 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] -(18) CometBroadcastHashJoin +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] + +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] +Right output [2]: [s_store_sk#15, s_state#17] Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) +(21) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22], [i_item_id#18, s_state#16, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] -(25) CometHashAggregate -Input [6]: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(26) CometExchange -Input [10]: [i_item_id#18, s_state#16, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Arguments: hashpartitioning(i_item_id#18, s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] (27) CometHashAggregate -Input [10]: [i_item_id#18, s_state#16, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] - -(28) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Arguments: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] - -(29) CometFilter -Input [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Condition : ((isnotnull(ss_cdemo_sk#32) AND isnotnull(ss_store_sk#33)) AND isnotnull(ss_item_sk#31)) - -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#39] - -(31) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Right output [1]: [cd_demo_sk#39] -Arguments: [ss_cdemo_sk#32], [cd_demo_sk#39], Inner, BuildRight - -(32) CometProject -Input [9]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38, cd_demo_sk#39] -Arguments: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38], [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] - -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#40] - -(34) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Right output [1]: [d_date_sk#40] -Arguments: [ss_sold_date_sk#38], [d_date_sk#40], Inner, BuildRight - -(35) CometProject -Input [8]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38, d_date_sk#40] -Arguments: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37], [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] - -(36) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#41, s_state#42] -Arguments: [s_store_sk#41, s_state#42] - -(37) CometFilter -Input [2]: [s_store_sk#41, s_state#42] -Condition : ((isnotnull(s_state#42) AND (s_state#42 = TN)) AND isnotnull(s_store_sk#41)) - -(38) CometProject -Input [2]: [s_store_sk#41, s_state#42] -Arguments: [s_store_sk#41], [s_store_sk#41] - -(39) CometBroadcastExchange -Input [1]: [s_store_sk#41] -Arguments: [s_store_sk#41] - -(40) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] -Right output [1]: [s_store_sk#41] -Arguments: [ss_store_sk#33], [s_store_sk#41], Inner, BuildRight - -(41) CometProject -Input [7]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, s_store_sk#41] -Arguments: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37], [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] - -(42) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#43, i_item_id#44] - -(43) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] -Right output [2]: [i_item_sk#43, i_item_id#44] -Arguments: [ss_item_sk#31], [i_item_sk#43], Inner, BuildRight - -(44) CometProject -Input [7]: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, i_item_sk#43, i_item_id#44] -Arguments: [i_item_id#44, agg1#45, agg2#46, agg3#47, agg4#48], [i_item_id#44, ss_quantity#34 AS agg1#45, ss_list_price#35 AS agg2#46, ss_coupon_amt#37 AS agg3#47, ss_sales_price#36 AS agg4#48] - -(45) CometHashAggregate -Input [5]: [i_item_id#44, agg1#45, agg2#46, agg3#47, agg4#48] -Keys [1]: [i_item_id#44] -Functions [4]: [partial_avg(agg1#45), partial_avg(UnscaledValue(agg2#46)), partial_avg(UnscaledValue(agg3#47)), partial_avg(UnscaledValue(agg4#48))] - -(46) CometExchange -Input [9]: [i_item_id#44, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56] -Arguments: hashpartitioning(i_item_id#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(28) CometExchange +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(30) CometNativeScan parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(31) CometFilter +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) + +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#43] + +(33) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight + +(34) CometProject +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] + +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#44] + +(36) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight + +(37) CometProject +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(38) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_state#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(39) CometFilter +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#46, 2, true, false, true) = TN) AND isnotnull(s_store_sk#45)) + +(40) CometProject +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] + +(41) CometBroadcastExchange +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] + +(42) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight + +(43) CometProject +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(44) ReusedExchange [Reuses operator id: 24] +Output [2]: [i_item_sk#47, i_item_id#20] + +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#20] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight + +(46) CometProject +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#20] +Arguments: [i_item_id#20, agg1#48, agg2#49, agg3#50, agg4#51], [i_item_id#20, ss_quantity#38 AS agg1#48, ss_list_price#39 AS agg2#49, ss_coupon_amt#41 AS agg3#50, ss_sales_price#40 AS agg4#51] (47) CometHashAggregate -Input [9]: [i_item_id#44, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56] -Keys [1]: [i_item_id#44] -Functions [4]: [avg(agg1#45), avg(UnscaledValue(agg2#46)), avg(UnscaledValue(agg3#47)), avg(UnscaledValue(agg4#48))] - -(48) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Arguments: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] - -(49) CometFilter -Input [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Condition : ((isnotnull(ss_cdemo_sk#58) AND isnotnull(ss_store_sk#59)) AND isnotnull(ss_item_sk#57)) - -(50) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#65] - -(51) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Right output [1]: [cd_demo_sk#65] -Arguments: [ss_cdemo_sk#58], [cd_demo_sk#65], Inner, BuildRight - -(52) CometProject -Input [9]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64, cd_demo_sk#65] -Arguments: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64], [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] - -(53) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#66] - -(54) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Right output [1]: [d_date_sk#66] -Arguments: [ss_sold_date_sk#64], [d_date_sk#66], Inner, BuildRight - -(55) CometProject -Input [8]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64, d_date_sk#66] -Arguments: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63], [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] - -(56) ReusedExchange [Reuses operator id: 39] -Output [1]: [s_store_sk#67] +Input [5]: [i_item_id#20, agg1#48, agg2#49, agg3#50, agg4#51] +Keys [1]: [i_item_id#20] +Functions [4]: [partial_avg(agg1#48), partial_avg(UnscaledValue(agg2#49)), partial_avg(UnscaledValue(agg3#50)), partial_avg(UnscaledValue(agg4#51))] + +(48) CometExchange +Input [9]: [i_item_id#20, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(49) CometHashAggregate +Input [9]: [i_item_id#20, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59] +Keys [1]: [i_item_id#20] +Functions [4]: [avg(agg1#48), avg(UnscaledValue(agg2#49)), avg(UnscaledValue(agg3#50)), avg(UnscaledValue(agg4#51))] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#67)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Condition : ((isnotnull(ss_cdemo_sk#61) AND isnotnull(ss_store_sk#62)) AND isnotnull(ss_item_sk#60)) + +(52) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#68] + +(53) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Right output [1]: [cd_demo_sk#68] +Arguments: [ss_cdemo_sk#61], [cd_demo_sk#68], Inner, BuildRight + +(54) CometProject +Input [9]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67, cd_demo_sk#68] +Arguments: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67], [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] + +(55) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#69] + +(56) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Right output [1]: [d_date_sk#69] +Arguments: [ss_sold_date_sk#67], [d_date_sk#69], Inner, BuildRight + +(57) CometProject +Input [8]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67, d_date_sk#69] +Arguments: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66], [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] + +(58) ReusedExchange [Reuses operator id: 41] +Output [1]: [s_store_sk#70] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] +Right output [1]: [s_store_sk#70] +Arguments: [ss_store_sk#62], [s_store_sk#70], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, s_store_sk#70] +Arguments: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66], [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] + +(61) CometNativeScan parquet spark_catalog.default.item +Output [1]: [i_item_sk#71] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(62) CometFilter +Input [1]: [i_item_sk#71] +Condition : isnotnull(i_item_sk#71) + +(63) CometBroadcastExchange +Input [1]: [i_item_sk#71] +Arguments: [i_item_sk#71] + +(64) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] +Right output [1]: [i_item_sk#71] +Arguments: [ss_item_sk#60], [i_item_sk#71], Inner, BuildRight + +(65) CometProject +Input [6]: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, i_item_sk#71] +Arguments: [agg1#72, agg2#73, agg3#74, agg4#75], [ss_quantity#63 AS agg1#72, ss_list_price#64 AS agg2#73, ss_coupon_amt#66 AS agg3#74, ss_sales_price#65 AS agg4#75] -(57) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] -Right output [1]: [s_store_sk#67] -Arguments: [ss_store_sk#59], [s_store_sk#67], Inner, BuildRight - -(58) CometProject -Input [7]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, s_store_sk#67] -Arguments: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63], [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] - -(59) CometNativeScan: `spark_catalog`.`default`.`item` -Output [1]: [i_item_sk#68] -Arguments: [i_item_sk#68] - -(60) CometFilter -Input [1]: [i_item_sk#68] -Condition : isnotnull(i_item_sk#68) - -(61) CometBroadcastExchange -Input [1]: [i_item_sk#68] -Arguments: [i_item_sk#68] - -(62) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] -Right output [1]: [i_item_sk#68] -Arguments: [ss_item_sk#57], [i_item_sk#68], Inner, BuildRight - -(63) CometProject -Input [6]: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, i_item_sk#68] -Arguments: [agg1#69, agg2#70, agg3#71, agg4#72], [ss_quantity#60 AS agg1#69, ss_list_price#61 AS agg2#70, ss_coupon_amt#63 AS agg3#71, ss_sales_price#62 AS agg4#72] - -(64) CometHashAggregate -Input [4]: [agg1#69, agg2#70, agg3#71, agg4#72] +(66) CometHashAggregate +Input [4]: [agg1#72, agg2#73, agg3#74, agg4#75] Keys: [] -Functions [4]: [partial_avg(agg1#69), partial_avg(UnscaledValue(agg2#70)), partial_avg(UnscaledValue(agg3#71)), partial_avg(UnscaledValue(agg4#72))] +Functions [4]: [partial_avg(agg1#72), partial_avg(UnscaledValue(agg2#73)), partial_avg(UnscaledValue(agg3#74)), partial_avg(UnscaledValue(agg4#75))] -(65) CometExchange -Input [8]: [sum#73, count#74, sum#75, count#76, sum#77, count#78, sum#79, count#80] +(67) CometExchange +Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(66) CometHashAggregate -Input [8]: [sum#73, count#74, sum#75, count#76, sum#77, count#78, sum#79, count#80] +(68) CometHashAggregate +Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] Keys: [] -Functions [4]: [avg(agg1#69), avg(UnscaledValue(agg2#70)), avg(UnscaledValue(agg3#71)), avg(UnscaledValue(agg4#72))] +Functions [4]: [avg(agg1#72), avg(UnscaledValue(agg2#73)), avg(UnscaledValue(agg3#74)), avg(UnscaledValue(agg4#75))] -(67) CometUnion -Child 0 Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] -Child 1 Input [7]: [i_item_id#44, s_state#86, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91] -Child 2 Input [7]: [i_item_id#92, s_state#93, g_state#94, agg1#95, agg2#96, agg3#97, agg4#98] +(69) CometUnion +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] +Child 1 Input [7]: [i_item_id#20, s_state#89, g_state#90, agg1#91, agg2#92, agg3#93, agg4#94] +Child 2 Input [7]: [i_item_id#95, s_state#96, g_state#97, agg1#98, agg2#99, agg3#100, agg4#101] -(68) CometTakeOrderedAndProject -Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST,s_state#16 ASC NULLS FIRST], output=[i_item_id#18,s_state#16,g_state#81,agg1#82,agg2#83,agg3#84,agg4#85]), [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85], 100, [i_item_id#18 ASC NULLS FIRST, s_state#16 ASC NULLS FIRST], [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] +(70) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#84,agg1#85,agg2#86,agg3#87,agg4#88]), [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] -(69) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] +(71) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/extended.txt new file mode 100644 index 0000000000..180236b925 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/extended.txt @@ -0,0 +1,91 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 88 out of 88 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/simplified.txt index 93723f1654..a7b0657337 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] CometExchange [i_item_id,s_state] #1 - CometHashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] @@ -15,24 +15,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] CometExchange [i_item_id] #6 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] @@ -42,17 +44,17 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] ReusedExchange [cd_demo_sk] #2 ReusedExchange [d_date_sk] #3 CometBroadcastExchange [s_store_sk] #7 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] CometExchange #8 - CometHashAggregate [sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] + CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] @@ -62,10 +64,10 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] ReusedExchange [cd_demo_sk] #2 ReusedExchange [d_date_sk] #3 ReusedExchange [s_store_sk] #7 CometBroadcastExchange [i_item_sk] #9 CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/explain.txt index 15052b5542..7e284f613e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,35 +15,43 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) + : : +- CometNativeScan parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] @@ -66,9 +74,12 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#9, s_county#10] @@ -91,13 +102,16 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -134,35 +148,42 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(25) CometNativeScan: `spark_catalog`.`default`.`customer` +(25) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (26) CometFilter Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Condition : isnotnull(c_customer_sk#17) -(27) CometBroadcastExchange +(27) CometProject Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(28) CometBroadcastHashJoin +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/simplified.txt index 726262ac46..cb3f282260 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] @@ -16,19 +16,20 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/explain.txt index 5bad8f9a0c..50f044bf92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/explain.txt @@ -1,60 +1,77 @@ == Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] @@ -86,12 +103,15 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#13] @@ -105,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -118,92 +138,132 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct -(21) BroadcastHashJoin [codegen id : 5] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None -(22) Filter [codegen id : 5] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(23) Project [codegen id : 5] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_state#16] -Arguments: [ca_address_sk#15, ca_state#16] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : isnotnull(ca_address_sk#15) +(31) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] -(26) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#15, ca_state#16] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] -(27) BroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(28) BroadcastHashJoin [codegen id : 5] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(29) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#16] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#16] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(31) CometFilter -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Condition : isnotnull(cd_demo_sk#17) +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(33) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(34) BroadcastHashJoin [codegen id : 5] +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None -(35) Project [codegen id : 5] -Output [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Input [8]: [c_current_cdemo_sk#4, ca_state#16, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(36) HashAggregate [codegen id : 5] -Input [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#20), partial_max(cd_dep_count#20), partial_sum(cd_dep_count#20), partial_avg(cd_dep_employed_count#21), partial_max(cd_dep_employed_count#21), partial_sum(cd_dep_employed_count#21), partial_avg(cd_dep_college_count#22), partial_max(cd_dep_college_count#22), partial_sum(cd_dep_college_count#22)] -Aggregate Attributes [13]: [count#23, sum#24, count#25, max#26, sum#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35] -Results [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, sum#37, count#38, max#39, sum#40, sum#41, count#42, max#43, sum#44, sum#45, count#46, max#47, sum#48] - -(37) Exchange -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, sum#37, count#38, max#39, sum#40, sum#41, count#42, max#43, sum#44, sum#45, count#46, max#47, sum#48] -Arguments: hashpartitioning(ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, sum#37, count#38, max#39, sum#40, sum#41, count#42, max#43, sum#44, sum#45, count#46, max#47, sum#48] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [count(1), avg(cd_dep_count#20), max(cd_dep_count#20), sum(cd_dep_count#20), avg(cd_dep_employed_count#21), max(cd_dep_employed_count#21), sum(cd_dep_employed_count#21), avg(cd_dep_college_count#22), max(cd_dep_college_count#22), sum(cd_dep_college_count#22)] -Aggregate Attributes [10]: [count(1)#49, avg(cd_dep_count#20)#50, max(cd_dep_count#20)#51, sum(cd_dep_count#20)#52, avg(cd_dep_employed_count#21)#53, max(cd_dep_employed_count#21)#54, sum(cd_dep_employed_count#21)#55, avg(cd_dep_college_count#22)#56, max(cd_dep_college_count#22)#57, sum(cd_dep_college_count#22)#58] -Results [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, count(1)#49 AS cnt1#59, avg(cd_dep_count#20)#50 AS avg(cd_dep_count)#60, max(cd_dep_count#20)#51 AS max(cd_dep_count)#61, sum(cd_dep_count#20)#52 AS sum(cd_dep_count)#62, cd_dep_employed_count#21, count(1)#49 AS cnt2#63, avg(cd_dep_employed_count#21)#53 AS avg(cd_dep_employed_count)#64, max(cd_dep_employed_count#21)#54 AS max(cd_dep_employed_count)#65, sum(cd_dep_employed_count#21)#55 AS sum(cd_dep_employed_count)#66, cd_dep_college_count#22, count(1)#49 AS cnt3#67, avg(cd_dep_college_count#22)#56 AS avg(cd_dep_college_count)#68, max(cd_dep_college_count#22)#57 AS max(cd_dep_college_count)#69, sum(cd_dep_college_count#22)#58 AS sum(cd_dep_college_count)#70] - -(39) TakeOrderedAndProject -Input [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cnt1#59, avg(cd_dep_count)#60, max(cd_dep_count)#61, sum(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, avg(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, sum(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, avg(cd_dep_college_count)#68, max(cd_dep_college_count)#69, sum(cd_dep_college_count)#70] -Arguments: 100, [ca_state#16 ASC NULLS FIRST, cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_dep_count#20 ASC NULLS FIRST, cd_dep_employed_count#21 ASC NULLS FIRST, cd_dep_college_count#22 ASC NULLS FIRST], [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cnt1#59, avg(cd_dep_count)#60, max(cd_dep_count)#61, sum(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, avg(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, sum(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, avg(cd_dep_college_count)#68, max(cd_dep_college_count)#69, sum(cd_dep_college_count)#70] +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#23)#55, max(cd_dep_count#23)#56, sum(cd_dep_count#23)#57, avg(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, sum(cd_dep_employed_count#24)#60, avg(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, sum(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, count(1)#54 AS cnt1#64, avg(cd_dep_count#23)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#23)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#24)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#24)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#25)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#25)#63 AS sum(cd_dep_college_count)#75] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/simplified.txt index 7bd4e9bfca..4b9be3b971 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/simplified.txt @@ -1,53 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/explain.txt index 9fda42eeea..39b6ecf9b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/explain.txt @@ -1,59 +1,70 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometHashAggregate (36) - +- CometExchange (35) - +- CometHashAggregate (34) - +- CometProject (33) - +- CometBroadcastHashJoin (32) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) : :- CometProject (23) : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) + : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (12) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (16) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- CometBroadcastExchange (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (29) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] -Arguments: [d_date_sk#6, d_year#7, d_qoy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct (5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] @@ -85,9 +96,12 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometNativeScan: `spark_catalog`.`default`.`web_sales` +(12) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#11] @@ -101,9 +115,12 @@ Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] Arguments: [customsk#12], [ws_bill_customer_sk#9 AS customsk#12] -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(16) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Arguments: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] Output [1]: [d_date_sk#15] @@ -134,66 +151,80 @@ Arguments: [c_customer_sk#1], [customsk#12], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` +(24) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] -Arguments: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (25) CometFilter Input [2]: [ca_address_sk#17, ca_state#18] Condition : isnotnull(ca_address_sk#17) -(26) CometBroadcastExchange +(26) CometProject Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] -(27) CometBroadcastHashJoin +(27) CometBroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: [ca_address_sk#17, ca_state#19] + +(28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [2]: [ca_address_sk#17, ca_state#18] +Right output [2]: [ca_address_sk#17, ca_state#19] Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight -(28) CometProject -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#18] -Arguments: [c_current_cdemo_sk#2, ca_state#18], [c_current_cdemo_sk#2, ca_state#18] - -(29) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(29) CometProject +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#19] +Arguments: [c_current_cdemo_sk#2, ca_state#19], [c_current_cdemo_sk#2, ca_state#19] -(30) CometFilter -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) +(30) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(31) CometBroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(31) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(32) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, ca_state#18] -Right output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight +(32) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(33) CometProject -Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(33) CometBroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(34) CometHashAggregate -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] +(34) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, ca_state#19] +Right output [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#20], Inner, BuildRight -(35) CometExchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(35) CometProject +Input [8]: [c_current_cdemo_sk#2, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (36) CometHashAggregate -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] + +(37) CometExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(38) CometHashAggregate +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] -(37) CometTakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#18 ASC NULLS FIRST,cd_gender#20 ASC NULLS FIRST,cd_marital_status#21 ASC NULLS FIRST,cd_dep_count#22 ASC NULLS FIRST,cd_dep_employed_count#23 ASC NULLS FIRST,cd_dep_college_count#24 ASC NULLS FIRST], output=[ca_state#18,cd_gender#20,cd_marital_status#21,cd_dep_count#22,cnt1#38,avg(cd_dep_count)#39,max(cd_dep_count)#40,sum(cd_dep_count)#41,cd_dep_employed_count#23,cnt2#42,avg(cd_dep_employed_count)#43,max(cd_dep_employed_count)#44,sum(cd_dep_employed_count)#45,cd_dep_college_count#24,cnt3#46,avg(cd_dep_college_count)#47,max(cd_dep_college_count)#48,sum(cd_dep_college_count)#49]), [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49], 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] +(39) CometTakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#19 ASC NULLS FIRST,cd_gender#26 ASC NULLS FIRST,cd_marital_status#27 ASC NULLS FIRST,cd_dep_count#23 ASC NULLS FIRST,cd_dep_employed_count#24 ASC NULLS FIRST,cd_dep_college_count#25 ASC NULLS FIRST], output=[ca_state#19,cd_gender#26,cd_marital_status#27,cd_dep_count#23,cnt1#41,avg(cd_dep_count)#42,max(cd_dep_count)#43,sum(cd_dep_count)#44,cd_dep_employed_count#24,cnt2#45,avg(cd_dep_employed_count)#46,max(cd_dep_employed_count)#47,sum(cd_dep_employed_count)#48,cd_dep_college_count#25,cnt3#49,avg(cd_dep_college_count)#50,max(cd_dep_college_count)#51,sum(cd_dep_college_count)#52]), [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52], 100, 0, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] -(38) ColumnarToRow [codegen id : 1] -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] +(40) CometColumnarToRow [codegen id : 1] +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/extended.txt new file mode 100644 index 0000000000..42c478c7c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/simplified.txt index b1134d0f8d..86604080fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometHashAggregate [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] @@ -13,28 +13,30 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [customsk] #4 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_customer_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 CometBroadcastExchange [ca_address_sk,ca_state] #5 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/explain.txt index 5f6d368556..ebb00d4d6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/explain.txt @@ -1,59 +1,67 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- Window (40) - +- * ColumnarToRow (39) - +- CometSort (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometExchange (35) - +- CometHashAggregate (34) - +- CometUnion (33) - :- CometHashAggregate (22) - : +- CometExchange (21) - : +- CometHashAggregate (20) - : +- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (43) ++- * Project (42) + +- Window (41) + +- * CometColumnarToRow (40) + +- CometSort (39) + +- CometExchange (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometUnion (34) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - : +- CometBroadcastExchange (17) - : +- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometHashAggregate (24) - : +- ReusedExchange (23) - +- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometHashAggregate (29) - +- ReusedExchange (28) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometHashAggregate (30) + +- ReusedExchange (29) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6, d_year#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#6, d_year#7] @@ -76,149 +84,159 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] + +(13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11, s_state#12] - -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(20) CometHashAggregate -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Keys [2]: [i_category#10, i_class#9] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) + +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] + +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] + +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight + +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(21) CometHashAggregate +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(21) CometExchange -Input [4]: [i_category#10, i_class#9, sum#13, sum#14] -Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(22) CometExchange +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [4]: [i_category#10, i_class#9, sum#13, sum#14] -Keys [2]: [i_category#10, i_class#9] +(23) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -(23) ReusedExchange [Reuses operator id: 21] -Output [4]: [i_category#15, i_class#16, sum#17, sum#18] +(24) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#17, sum#18] -(24) CometHashAggregate -Input [4]: [i_category#15, i_class#16, sum#17, sum#18] -Keys [2]: [i_category#15, i_class#16] +(25) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#17, sum#18] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#19)), sum(UnscaledValue(ss_ext_sales_price#20))] -(25) CometHashAggregate -Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#15] -Keys [1]: [i_category#15] +(26) CometHashAggregate +Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#12] +Keys [1]: [i_category#12] Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] -(26) CometExchange -Input [5]: [i_category#15, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(i_category#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometExchange +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(i_category#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(27) CometHashAggregate -Input [5]: [i_category#15, sum#23, isEmpty#24, sum#25, isEmpty#26] -Keys [1]: [i_category#15] +(28) CometHashAggregate +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [i_category#12] Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] -(28) ReusedExchange [Reuses operator id: 21] -Output [4]: [i_category#27, i_class#28, sum#29, sum#30] - -(29) CometHashAggregate -Input [4]: [i_category#27, i_class#28, sum#29, sum#30] -Keys [2]: [i_category#27, i_class#28] -Functions [2]: [sum(UnscaledValue(ss_net_profit#31)), sum(UnscaledValue(ss_ext_sales_price#32))] +(29) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#27, sum#28] (30) CometHashAggregate -Input [2]: [ss_net_profit#33, ss_ext_sales_price#34] +Input [4]: [i_category#12, i_class#11, sum#27, sum#28] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] + +(31) CometHashAggregate +Input [2]: [ss_net_profit#31, ss_ext_sales_price#32] Keys: [] -Functions [2]: [partial_sum(ss_net_profit#33), partial_sum(ss_ext_sales_price#34)] +Functions [2]: [partial_sum(ss_net_profit#31), partial_sum(ss_ext_sales_price#32)] -(31) CometExchange -Input [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] +(32) CometExchange +Input [4]: [sum#33, isEmpty#34, sum#35, isEmpty#36] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometHashAggregate -Input [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] +(33) CometHashAggregate +Input [4]: [sum#33, isEmpty#34, sum#35, isEmpty#36] Keys: [] -Functions [2]: [sum(ss_net_profit#33), sum(ss_ext_sales_price#34)] +Functions [2]: [sum(ss_net_profit#31), sum(ss_ext_sales_price#32)] -(33) CometUnion -Child 0 Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Child 1 Input [6]: [gross_margin#43, i_category#15, i_class#44, t_category#45, t_class#46, lochierarchy#47] +(34) CometUnion +Child 0 Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Child 1 Input [6]: [gross_margin#43, i_category#12, i_class#44, t_category#45, t_class#46, lochierarchy#47] Child 2 Input [6]: [gross_margin#48, i_category#49, i_class#50, t_category#51, t_class#52, lochierarchy#53] -(34) CometHashAggregate -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Keys [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] +(35) CometHashAggregate +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Keys [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] Functions: [] -(35) CometExchange -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Arguments: hashpartitioning(gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(36) CometExchange +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Arguments: hashpartitioning(gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(36) CometHashAggregate -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Keys [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] +(37) CometHashAggregate +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Keys [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] Functions: [] -(37) CometExchange -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] +(38) CometExchange +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] Arguments: hashpartitioning(lochierarchy#42, _w0#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(38) CometSort -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] -Arguments: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54], [lochierarchy#42 ASC NULLS FIRST, _w0#54 ASC NULLS FIRST, gross_margin#39 ASC NULLS FIRST] +(39) CometSort +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] +Arguments: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54], [lochierarchy#42 ASC NULLS FIRST, _w0#54 ASC NULLS FIRST, gross_margin#37 ASC NULLS FIRST] -(39) ColumnarToRow [codegen id : 1] -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] +(40) CometColumnarToRow [codegen id : 1] +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] -(40) Window -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] -Arguments: [rank(gross_margin#39) windowspecdefinition(lochierarchy#42, _w0#54, gross_margin#39 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#55], [lochierarchy#42, _w0#54], [gross_margin#39 ASC NULLS FIRST] +(41) Window +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] +Arguments: [rank(gross_margin#37) windowspecdefinition(lochierarchy#42, _w0#54, gross_margin#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#55], [lochierarchy#42, _w0#54], [gross_margin#37 ASC NULLS FIRST] -(41) Project [codegen id : 2] -Output [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] -Input [6]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54, rank_within_parent#55] +(42) Project [codegen id : 2] +Output [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] +Input [6]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54, rank_within_parent#55] -(42) TakeOrderedAndProject -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] -Arguments: 100, [lochierarchy#42 DESC NULLS LAST, CASE WHEN (lochierarchy#42 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#55 ASC NULLS FIRST], [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] +(43) TakeOrderedAndProject +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] +Arguments: 100, [lochierarchy#42 DESC NULLS LAST, CASE WHEN (lochierarchy#42 = 0) THEN i_category#38 END ASC NULLS FIRST, rank_within_parent#55 ASC NULLS FIRST], [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/extended.txt new file mode 100644 index 0000000000..30d5d43026 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/extended.txt @@ -0,0 +1,87 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 81 out of 84 eligible operators (96%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/simplified.txt index f0d5567dc4..62746a00f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/simplified.txt @@ -4,17 +4,17 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i InputAdapter Window [gross_margin,lochierarchy,_w0] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] CometExchange [lochierarchy,_w0] #1 - CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,t_category,t_class] + CometHashAggregate [t_category] [gross_margin,i_category,i_class,lochierarchy,_w0,t_class] CometExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] CometUnion [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),i_category,i_class] CometExchange [i_category,i_class] #3 - CometHashAggregate [i_category,i_class,sum,sum,ss_net_profit,ss_ext_sales_price] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,sum,sum] CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] @@ -22,25 +22,26 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] CometExchange [i_category] #7 - CometHashAggregate [i_category,sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] - CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,i_category,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] CometExchange #8 - CometHashAggregate [sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] - CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_category,i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/explain.txt index 37d9b5e7a8..50eea35380 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/explain.txt @@ -1,243 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight - -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(18) CometHashAggregate -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] - -(19) CometExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] - -(21) CometExchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] - -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#17], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Arguments: [avg(_w0#16) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] - -(27) Filter [codegen id : 7] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] -Condition : ((isnotnull(avg_monthly_sales#18) AND (avg_monthly_sales#18 > 0.000000)) AND CASE WHEN (avg_monthly_sales#18 > 0.000000) THEN ((abs((sum_sales#15 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] - -(29) ReusedExchange [Reuses operator id: 19] -Output [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] - -(30) CometHashAggregate -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -Keys [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) + +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight + +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] + +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] + +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] + +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] + +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] + +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) + +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] + +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(31) CometExchange -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: hashpartitioning(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15], [i_category#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(34) Window -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#19, i_brand#20, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#15 AS sum_sales#28, rn#27] -Input [8]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15, rn#27] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(36) BroadcastExchange -Input [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] -(39) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(40) CometSort -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15], [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(42) Window -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(43) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#15 AS sum_sales#36, rn#35] -Input [8]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15, rn#35] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] -(44) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(45) BroadcastExchange +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, (rn#35 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, sum_sales#28 AS psum#37, sum_sales#36 AS nsum#38] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28, i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(47) Project [codegen id : 7] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] -(47) TakeOrderedAndProject -Input [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] -Arguments: 100, [(sum_sales#15 - avg_monthly_sales#18) ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] +(48) TakeOrderedAndProject +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/extended.txt new file mode 100644 index 0000000000..eca90ab6dc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/simplified.txt index 196a6d2c7d..4f03ba59f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/explain.txt index ef09c3f299..cba2ffa7be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/explain.txt @@ -1,56 +1,91 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- Union (43) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometExchange (19) - : +- CometHashAggregate (18) - : +- CometExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - :- * Project (34) - : +- * Filter (33) - : +- Window (32) - : +- * Sort (31) - : +- Window (30) - : +- * ColumnarToRow (29) - : +- CometSort (28) - : +- ReusedExchange (27) - +- * Project (42) - +- * Filter (41) - +- Window (40) - +- * Sort (39) - +- Window (38) - +- * ColumnarToRow (37) - +- CometSort (36) - +- ReusedExchange (35) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] +PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] @@ -64,9 +99,12 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometNativeScan: `spark_catalog`.`default`.`web_returns` +(5) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct (6) CometFilter Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] @@ -85,9 +123,12 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(10) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_moy#14] @@ -132,7 +173,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(21) ColumnarToRow [codegen id : 1] +(21) CometColumnarToRow [codegen id : 1] Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window @@ -155,87 +196,230 @@ Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(27) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#27, return_ratio#28, currency_ratio#29] - -(28) CometSort -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [item#27, return_ratio#28, currency_ratio#29], [return_ratio#28 ASC NULLS FIRST] - -(29) ColumnarToRow [codegen id : 4] -Input [3]: [item#27, return_ratio#28, currency_ratio#29] - -(30) Window -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [rank(return_ratio#28) windowspecdefinition(return_ratio#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#30], [return_ratio#28 ASC NULLS FIRST] - -(31) Sort [codegen id : 5] -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [currency_ratio#29 ASC NULLS FIRST], false, 0 - -(32) Window -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [rank(currency_ratio#29) windowspecdefinition(currency_ratio#29 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#31], [currency_ratio#29 ASC NULLS FIRST] - -(33) Filter [codegen id : 6] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] -Condition : ((return_rank#30 <= 10) OR (currency_rank#31 <= 10)) - -(34) Project [codegen id : 6] -Output [5]: [catalog AS channel#32, item#27, return_ratio#28, return_rank#30, currency_rank#31] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] - -(35) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#33, return_ratio#34, currency_ratio#35] - -(36) CometSort -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [item#33, return_ratio#34, currency_ratio#35], [return_ratio#34 ASC NULLS FIRST] - -(37) ColumnarToRow [codegen id : 7] -Input [3]: [item#33, return_ratio#34, currency_ratio#35] - -(38) Window -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [rank(return_ratio#34) windowspecdefinition(return_ratio#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#36], [return_ratio#34 ASC NULLS FIRST] - -(39) Sort [codegen id : 8] -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [currency_ratio#35 ASC NULLS FIRST], false, 0 - -(40) Window -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [rank(currency_ratio#35) windowspecdefinition(currency_ratio#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#37], [currency_ratio#35 ASC NULLS FIRST] - -(41) Filter [codegen id : 9] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] -Condition : ((return_rank#36 <= 10) OR (currency_rank#37 <= 10)) - -(42) Project [codegen id : 9] -Output [5]: [store AS channel#38, item#33, return_ratio#34, return_rank#36, currency_rank#37] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] - -(43) Union - -(44) HashAggregate [codegen id : 10] +(27) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] +PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) + +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(31) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] +PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) + +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(54) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(45) Exchange +(75) CometColumnarExchange Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(46) HashAggregate [codegen id : 11] +(76) CometHashAggregate Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(47) TakeOrderedAndProject +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: 100, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/extended.txt new file mode 100644 index 0000000000..262571ed59 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/extended.txt @@ -0,0 +1,86 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- Filter + +- Window + +- Sort + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 63 out of 80 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/simplified.txt index baf07c5c9a..117b74a75b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/simplified.txt @@ -1,69 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (11) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/explain.txt index 7044110296..e9fc524fbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/explain.txt @@ -1,65 +1,100 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * Filter (47) - +- * HashAggregate (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- Window (37) - : +- * Sort (36) - : +- Exchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * SortMergeJoin FullOuter (32) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- Exchange (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (16) - : : : +- Window (15) - : : : +- * ColumnarToRow (14) - : : : +- CometSort (13) - : : : +- CometExchange (12) - : : : +- CometHashAggregate (11) - : : : +- CometExchange (10) - : : : +- CometHashAggregate (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * ColumnarToRow (19) - : : +- CometSort (18) - : : +- ReusedExchange (17) - : +- * Sort (31) - : +- ReusedExchange (30) - +- BroadcastExchange (42) - +- * Project (41) - +- Window (40) - +- * Sort (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` +TakeOrderedAndProject (76) ++- * Filter (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- Window (64) + : +- * CometColumnarToRow (63) + : +- CometSort (62) + : +- CometExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometSortMergeJoin (58) + : :- CometSort (30) + : : +- CometColumnarExchange (29) + : : +- * HashAggregate (28) + : : +- * CometColumnarToRow (27) + : : +- CometColumnarExchange (26) + : : +- * HashAggregate (25) + : : +- * Project (24) + : : +- * BroadcastHashJoin Inner BuildRight (23) + : : :- * Project (16) + : : : +- Window (15) + : : : +- * CometColumnarToRow (14) + : : : +- CometSort (13) + : : : +- CometExchange (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (22) + : : +- * Project (21) + : : +- Window (20) + : : +- * CometColumnarToRow (19) + : : +- CometSort (18) + : : +- ReusedExchange (17) + : +- CometSort (57) + : +- CometColumnarExchange (56) + : +- * HashAggregate (55) + : +- * CometColumnarToRow (54) + : +- CometColumnarExchange (53) + : +- * HashAggregate (52) + : +- * Project (51) + : +- * BroadcastHashJoin Inner BuildRight (50) + : :- * Project (43) + : : +- Window (42) + : : +- * CometColumnarToRow (41) + : : +- CometSort (40) + : : +- CometExchange (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometFilter (32) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (31) + : : +- ReusedExchange (33) + : +- BroadcastExchange (49) + : +- * Project (48) + : +- Window (47) + : +- * CometColumnarToRow (46) + : +- CometSort (45) + : +- ReusedExchange (44) + +- BroadcastExchange (70) + +- * Project (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- ReusedExchange (65) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] @@ -104,7 +139,7 @@ Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] Arguments: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(14) ColumnarToRow [codegen id : 1] +(14) CometColumnarToRow [codegen id : 1] Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] (15) Window @@ -122,7 +157,7 @@ Output [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] Arguments: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12], [ws_item_sk#12 ASC NULLS FIRST, d_date#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 2] +(19) CometColumnarToRow [codegen id : 2] Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (20) Window @@ -154,106 +189,225 @@ Functions [1]: [partial_sum(sumws#15)] Aggregate Attributes [2]: [sum#16, isEmpty#17] Results [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] -(26) Exchange +(26) CometColumnarExchange Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] -Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(27) HashAggregate [codegen id : 5] +(27) CometColumnarToRow [codegen id : 5] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] + +(28) HashAggregate [codegen id : 5] Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] Keys [3]: [item_sk#8, d_date#5, sumws#9] Functions [1]: [sum(sumws#15)] Aggregate Attributes [1]: [sum(sumws#15)#20] Results [3]: [item_sk#8, d_date#5, sum(sumws#15)#20 AS cume_sales#21] -(28) Exchange +(29) CometColumnarExchange Input [3]: [item_sk#8, d_date#5, cume_sales#21] -Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(29) Sort [codegen id : 6] +(30) CometSort Input [3]: [item_sk#8, d_date#5, cume_sales#21] -Arguments: [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(30) ReusedExchange [Reuses operator id: 28] -Output [3]: [item_sk#22, d_date#23, cume_sales#24] - -(31) Sort [codegen id : 12] -Input [3]: [item_sk#22, d_date#23, cume_sales#24] -Arguments: [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], false, 0 - -(32) SortMergeJoin [codegen id : 13] -Left keys [2]: [item_sk#8, d_date#5] -Right keys [2]: [item_sk#22, d_date#23] -Join type: FullOuter -Join condition: None - -(33) Filter [codegen id : 13] -Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#22, d_date#23, cume_sales#24] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#22 END) - -(34) Project [codegen id : 13] -Output [4]: [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#22 END AS item_sk#25, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#23 END AS d_date#26, cume_sales#21 AS web_sales#27, cume_sales#24 AS store_sales#28] -Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#22, d_date#23, cume_sales#24] - -(35) Exchange -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: hashpartitioning(item_sk#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(36) Sort [codegen id : 14] -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [item_sk#25 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], false, 0 - -(37) Window -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [row_number() windowspecdefinition(item_sk#25, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#29], [item_sk#25], [d_date#26 ASC NULLS FIRST] - -(38) ReusedExchange [Reuses operator id: 35] -Output [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] - -(39) Sort [codegen id : 28] -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [item_sk#25 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], false, 0 - -(40) Window -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [row_number() windowspecdefinition(item_sk#25, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [item_sk#25], [d_date#26 ASC NULLS FIRST] - -(41) Project [codegen id : 29] -Output [4]: [item_sk#25 AS item_sk#31, web_sales#27 AS web_sales#32, store_sales#28 AS store_sales#33, rk#30] -Input [5]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, rk#30] - -(42) BroadcastExchange -Input [4]: [item_sk#31, web_sales#32, store_sales#33, rk#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -(43) BroadcastHashJoin [codegen id : 30] -Left keys [1]: [item_sk#25] -Right keys [1]: [item_sk#31] +Arguments: [item_sk#8, d_date#5, cume_sales#21], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(31) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(32) CometFilter +Input [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_item_sk#22) + +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#25, d_date#26] + +(34) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Right output [2]: [d_date_sk#25, d_date#26] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight + +(35) CometProject +Input [5]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_date#26] +Arguments: [ss_item_sk#22, ss_sales_price#23, d_date#26], [ss_item_sk#22, ss_sales_price#23, d_date#26] + +(36) CometHashAggregate +Input [3]: [ss_item_sk#22, ss_sales_price#23, d_date#26] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#23))] + +(37) CometExchange +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Arguments: hashpartitioning(ss_item_sk#22, d_date#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(38) CometHashAggregate +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [sum(UnscaledValue(ss_sales_price#23))] + +(39) CometExchange +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: hashpartitioning(ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(40) CometSort +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22], [ss_item_sk#22 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] + +(41) CometColumnarToRow [codegen id : 6] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] + +(42) Window +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [row_number() windowspecdefinition(ss_item_sk#22, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [ss_item_sk#22], [d_date#26 ASC NULLS FIRST] + +(43) Project [codegen id : 9] +Output [4]: [item_sk#28, d_date#26, sumss#29, rk#30] +Input [5]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22, rk#30] + +(44) ReusedExchange [Reuses operator id: 39] +Output [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] + +(45) CometSort +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32], [ss_item_sk#32 ASC NULLS FIRST, d_date#31 ASC NULLS FIRST] + +(46) CometColumnarToRow [codegen id : 7] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] + +(47) Window +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [row_number() windowspecdefinition(ss_item_sk#32, d_date#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [ss_item_sk#32], [d_date#31 ASC NULLS FIRST] + +(48) Project [codegen id : 8] +Output [3]: [item_sk#28 AS item_sk#34, sumss#29 AS sumss#35, rk#33] +Input [5]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32, rk#33] + +(49) BroadcastExchange +Input [3]: [item_sk#34, sumss#35, rk#33] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] + +(50) BroadcastHashJoin [codegen id : 9] +Left keys [1]: [item_sk#28] +Right keys [1]: [item_sk#34] +Join type: Inner +Join condition: (rk#30 >= rk#33) + +(51) Project [codegen id : 9] +Output [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Input [7]: [item_sk#28, d_date#26, sumss#29, rk#30, item_sk#34, sumss#35, rk#33] + +(52) HashAggregate [codegen id : 9] +Input [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#36, isEmpty#37] +Results [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] + +(53) CometColumnarExchange +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Arguments: hashpartitioning(item_sk#28, d_date#26, sumss#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(54) CometColumnarToRow [codegen id : 10] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] + +(55) HashAggregate [codegen id : 10] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#40] +Results [3]: [item_sk#28, d_date#26, sum(sumss#35)#40 AS cume_sales#41] + +(56) CometColumnarExchange +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: hashpartitioning(item_sk#28, d_date#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(57) CometSort +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#28, d_date#26, cume_sales#41], [item_sk#28 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#21] +Right output [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#8, d_date#5], [item_sk#28, d_date#26], FullOuter + +(59) CometFilter +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END) + +(60) CometProject +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END AS item_sk#42, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#26 END AS d_date#43, cume_sales#21 AS web_sales#44, cume_sales#41 AS store_sales#45] + +(61) CometExchange +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: hashpartitioning(item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(62) CometSort +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] + +(63) CometColumnarToRow [codegen id : 11] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(64) Window +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#46], [item_sk#42], [d_date#43 ASC NULLS FIRST] + +(65) ReusedExchange [Reuses operator id: 61] +Output [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(66) CometSort +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 22] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(68) Window +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#47], [item_sk#42], [d_date#43 ASC NULLS FIRST] + +(69) Project [codegen id : 23] +Output [4]: [item_sk#42 AS item_sk#48, web_sales#44 AS web_sales#49, store_sales#45 AS store_sales#50, rk#47] +Input [5]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#47] + +(70) BroadcastExchange +Input [4]: [item_sk#48, web_sales#49, store_sales#50, rk#47] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(71) BroadcastHashJoin [codegen id : 24] +Left keys [1]: [item_sk#42] +Right keys [1]: [item_sk#48] Join type: Inner -Join condition: (rk#29 >= rk#30) - -(44) Project [codegen id : 30] -Output [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_sales#32, store_sales#33] -Input [9]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, rk#29, item_sk#31, web_sales#32, store_sales#33, rk#30] - -(45) HashAggregate [codegen id : 30] -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_sales#32, store_sales#33] -Keys [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Functions [2]: [partial_max(web_sales#32), partial_max(store_sales#33)] -Aggregate Attributes [2]: [max#34, max#35] -Results [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, max#36, max#37] - -(46) HashAggregate [codegen id : 30] -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, max#36, max#37] -Keys [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Functions [2]: [max(web_sales#32), max(store_sales#33)] -Aggregate Attributes [2]: [max(web_sales#32)#38, max(store_sales#33)#39] -Results [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, max(web_sales#32)#38 AS web_cumulative#40, max(store_sales#33)#39 AS store_cumulative#41] - -(47) Filter [codegen id : 30] -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_cumulative#40, store_cumulative#41] -Condition : ((isnotnull(web_cumulative#40) AND isnotnull(store_cumulative#41)) AND (web_cumulative#40 > store_cumulative#41)) - -(48) TakeOrderedAndProject -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_cumulative#40, store_cumulative#41] -Arguments: 100, [item_sk#25 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_cumulative#40, store_cumulative#41] +Join condition: (rk#46 >= rk#47) + +(72) Project [codegen id : 24] +Output [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Input [9]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#46, item_sk#48, web_sales#49, store_sales#50, rk#47] + +(73) HashAggregate [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [partial_max(web_sales#49), partial_max(store_sales#50)] +Aggregate Attributes [2]: [max#51, max#52] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] + +(74) HashAggregate [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [max(web_sales#49), max(store_sales#50)] +Aggregate Attributes [2]: [max(web_sales#49)#55, max(store_sales#50)#56] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max(web_sales#49)#55 AS web_cumulative#57, max(store_sales#50)#56 AS store_cumulative#58] + +(75) Filter [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Condition : ((isnotnull(web_cumulative#57) AND isnotnull(store_cumulative#58)) AND (web_cumulative#57 > store_cumulative#58)) + +(76) TakeOrderedAndProject +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Arguments: 100, [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/extended.txt new file mode 100644 index 0000000000..e32bdbf198 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/extended.txt @@ -0,0 +1,188 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- HashAggregate + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- CometColumnarToRow + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 126 out of 172 eligible operators (73%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/simplified.txt index 055420a2a4..ee1947e250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (30) + WholeStageCodegen (24) Filter [web_cumulative,store_cumulative] HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] @@ -7,69 +7,101 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store BroadcastHashJoin [item_sk,item_sk,rk,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (14) - Sort [item_sk,d_date] + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (13) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - Filter [item_sk,item_sk] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (5) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumws] #3 - WholeStageCodegen (4) - HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumws,sumws] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - CometExchange [ws_item_sk] #4 - CometHashAggregate [item_sk,d_date,sumws,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometExchange [ws_item_sk,d_date] #5 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [item_sk,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,cume_sales] #2 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (5) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumws] #3 + WholeStageCodegen (4) + HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumws,sumws] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + CometExchange [ws_item_sk] #4 + CometHashAggregate [sum] [item_sk,d_date,sumws,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #5 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + Project [item_sk,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #8 + WholeStageCodegen (10) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumss] #9 + WholeStageCodegen (9) + HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumss,sumss] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sum] [item_sk,d_date,sumss,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #11 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #6 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (8) + Project [item_sk,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (29) + BroadcastExchange #13 + WholeStageCodegen (23) Project [item_sk,web_sales,store_sales,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (28) - Sort [item_sk,d_date] + WholeStageCodegen (22) + CometColumnarToRow InputAdapter - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 + CometSort [item_sk,d_date,web_sales,store_sales] + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/explain.txt index 674d4c6f7b..0cd88caaff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/explain.txt @@ -1,243 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(18) CometHashAggregate -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] - -(19) CometExchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] - -(21) CometExchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] - -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#16], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Arguments: [avg(_w0#15) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#17], [i_category#3, i_brand#2, cc_name#12, d_year#9] - -(27) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] -Condition : ((isnotnull(avg_monthly_sales#17) AND (avg_monthly_sales#17 > 0.000000)) AND CASE WHEN (avg_monthly_sales#17 > 0.000000) THEN ((abs((sum_sales#14 - avg_monthly_sales#17)) / avg_monthly_sales#17) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] - -(29) ReusedExchange [Reuses operator id: 19] -Output [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] - -(30) CometHashAggregate -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] -Keys [5]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight + +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] + +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] + +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] + +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] + +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] + +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) + +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] + +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] + +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(31) CometExchange -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: hashpartitioning(i_category#18, i_brand#19, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14], [i_category#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(34) Window -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#18, i_brand#19, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#18, i_brand#19, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#14 AS sum_sales#26, rn#25] -Input [7]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14, rn#25] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(36) BroadcastExchange -Input [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#18, i_brand#19, cc_name#20, (rn#25 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] -(39) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(40) CometSort -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14], [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(42) Window -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(43) Window +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#14 AS sum_sales#33, rn#32] -Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14, rn#32] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] -(44) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(45) BroadcastExchange +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#32 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, sum_sales#26 AS psum#34, sum_sales#33 AS nsum#35] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26, i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(47) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] -(47) TakeOrderedAndProject -Input [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] -Arguments: 100, [(sum_sales#14 - avg_monthly_sales#17) ASC NULLS FIRST, d_year#9 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] +(48) TakeOrderedAndProject +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/extended.txt new file mode 100644 index 0000000000..357307b654 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.call_center + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/simplified.txt index b840a4a743..9656904abd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,cc_name] #1 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,cc_name] #7 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/explain.txt index 498f1c3292..9b7d6d0f22 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/explain.txt @@ -1,74 +1,97 @@ == Physical Plan == -TakeOrderedAndProject (65) -+- * HashAggregate (64) - +- Exchange (63) - +- * HashAggregate (62) - +- Union (61) - :- * HashAggregate (50) - : +- Exchange (49) - : +- * HashAggregate (48) - : +- Union (47) - : :- * HashAggregate (22) - : : +- * ColumnarToRow (21) - : : +- CometExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometUnion (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - : :- * HashAggregate (25) - : : +- * ColumnarToRow (24) - : : +- ReusedExchange (23) - : +- * HashAggregate (46) - : +- * ColumnarToRow (45) - : +- CometExchange (44) - : +- CometHashAggregate (43) - : +- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometUnion (36) - : : : :- CometProject (28) - : : : : +- CometFilter (27) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) - : : : +- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometBroadcastExchange (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (29) - : : : +- CometProject (33) - : : : +- CometFilter (32) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (31) - : : +- ReusedExchange (37) - : +- ReusedExchange (40) - :- * HashAggregate (55) - : +- Exchange (54) - : +- * HashAggregate (53) - : +- * HashAggregate (52) - : +- ReusedExchange (51) - +- * HashAggregate (60) - +- Exchange (59) - +- * HashAggregate (58) - +- * HashAggregate (57) - +- ReusedExchange (56) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (84) ++- CometTakeOrderedAndProject (83) + +- CometHashAggregate (82) + +- CometExchange (81) + +- CometHashAggregate (80) + +- CometUnion (79) + :- CometHashAggregate (68) + : +- CometExchange (67) + : +- CometHashAggregate (66) + : +- CometUnion (65) + : :- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : :- CometHashAggregate (41) + : : +- CometExchange (40) + : : +- CometHashAggregate (39) + : : +- CometProject (38) + : : +- CometBroadcastHashJoin (37) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometUnion (29) + : : : : :- CometProject (25) + : : : : : +- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (23) + : : : : +- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (26) + : : : +- ReusedExchange (30) + : : +- CometBroadcastExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) + : +- CometHashAggregate (64) + : +- CometExchange (63) + : +- CometHashAggregate (62) + : +- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometUnion (52) + : : : :- CometProject (44) + : : : : +- CometFilter (43) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (42) + : : : +- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastExchange (46) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (45) + : : : +- CometProject (49) + : : : +- CometFilter (48) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (59) + : +- CometProject (58) + : +- CometFilter (57) + : +- CometNativeScan parquet spark_catalog.default.web_site (56) + :- CometHashAggregate (73) + : +- CometExchange (72) + : +- CometHashAggregate (71) + : +- CometHashAggregate (70) + : +- ReusedExchange (69) + +- CometHashAggregate (78) + +- CometExchange (77) + +- CometHashAggregate (76) + +- CometHashAggregate (75) + +- ReusedExchange (74) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] @@ -78,9 +101,13 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct (5) CometFilter Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] @@ -94,9 +121,12 @@ Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, n Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21, d_date#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [d_date_sk#21, d_date#22] @@ -119,244 +149,333 @@ Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(14) CometNativeScan: `spark_catalog`.`default`.`store` +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (15) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(16) CometBroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] -(17) CometBroadcastHashJoin +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] +Right output [2]: [s_store_sk#23, s_store_id#25] Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(19) CometHashAggregate -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -(20) CometExchange -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) ColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Keys [1]: [s_store_id#24] +(22) CometHashAggregate +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#29, sum(UnscaledValue(return_amt#9))#30, sum(UnscaledValue(profit#8))#31, sum(UnscaledValue(net_loss#10))#32] -Results [5]: [store channel AS channel#33, concat(store, s_store_id#24) AS id#34, MakeDecimal(sum(UnscaledValue(sales_price#7))#29,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#9))#30,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#8))#31,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#32,17,2)) AS profit#37] - -(23) ReusedExchange [Reuses operator id: 20] -Output [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] -(24) ColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] - -(25) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] -Keys [1]: [cp_catalog_page_id#38] -Functions [4]: [sum(UnscaledValue(sales_price#43)), sum(UnscaledValue(return_amt#44)), sum(UnscaledValue(profit#45)), sum(UnscaledValue(net_loss#46))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#43))#47, sum(UnscaledValue(return_amt#44))#48, sum(UnscaledValue(profit#45))#49, sum(UnscaledValue(net_loss#46))#50] -Results [5]: [catalog channel AS channel#51, concat(catalog_page, cp_catalog_page_id#38) AS id#52, MakeDecimal(sum(UnscaledValue(sales_price#43))#47,17,2) AS sales#53, MakeDecimal(sum(UnscaledValue(return_amt#44))#48,17,2) AS returns#54, (MakeDecimal(sum(UnscaledValue(profit#45))#49,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#46))#50,17,2)) AS profit#55] - -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] +(23) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +PushedFilters: [IsNotNull(cs_catalog_page_sk)] +ReadSchema: struct + +(24) CometFilter +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : isnotnull(cs_catalog_page_sk#30) + +(25) CometProject +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39], [cs_catalog_page_sk#30 AS page_sk#34, cs_sold_date_sk#33 AS date_sk#35, cs_ext_sales_price#31 AS sales_price#36, cs_net_profit#32 AS profit#37, 0.00 AS return_amt#38, 0.00 AS net_loss#39] + +(26) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] +PushedFilters: [IsNotNull(cr_catalog_page_sk)] +ReadSchema: struct (27) CometFilter -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Condition : isnotnull(ws_web_site_sk#56) +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Condition : isnotnull(cr_catalog_page_sk#40) (28) CometProject -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65], [ws_web_site_sk#56 AS wsr_web_site_sk#60, ws_sold_date_sk#59 AS date_sk#61, ws_ext_sales_price#57 AS sales_price#62, ws_net_profit#58 AS profit#63, 0.00 AS return_amt#64, 0.00 AS net_loss#65] +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Arguments: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49], [cr_catalog_page_sk#40 AS page_sk#44, cr_returned_date_sk#43 AS date_sk#45, 0.00 AS sales_price#46, 0.00 AS profit#47, cr_return_amount#41 AS return_amt#48, cr_net_loss#42 AS net_loss#49] -(29) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] +(29) CometUnion +Child 0 Input [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Child 1 Input [6]: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49] -(30) CometBroadcastExchange -Input [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] +(30) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#50] -(31) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] +(31) CometBroadcastHashJoin +Left output [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [1]: [d_date_sk#50] +Arguments: [date_sk#35], [d_date_sk#50], Inner, BuildRight -(32) CometFilter -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_item_sk#71) AND isnotnull(ws_order_number#73)) AND isnotnull(ws_web_site_sk#72)) +(32) CometProject +Input [7]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39, d_date_sk#50] +Arguments: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39], [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] -(33) CometProject -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73], [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] +(33) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct -(34) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Right output [3]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wr_item_sk#66, wr_order_number#67], [ws_item_sk#71, ws_order_number#73], Inner, BuildLeft +(34) CometFilter +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Condition : isnotnull(cp_catalog_page_sk#51) (35) CometProject -Input [8]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70, ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80], [ws_web_site_sk#72 AS wsr_web_site_sk#75, wr_returned_date_sk#70 AS date_sk#76, 0.00 AS sales_price#77, 0.00 AS profit#78, wr_return_amt#68 AS return_amt#79, wr_net_loss#69 AS net_loss#80] - -(36) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Child 1 Input [6]: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80] - -(37) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#81] - -(38) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [1]: [d_date_sk#81] -Arguments: [date_sk#61], [d_date_sk#81], Inner, BuildRight - -(39) CometProject -Input [7]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65, d_date_sk#81] -Arguments: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65], [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] - -(40) ReusedExchange [Reuses operator id: 16] -Output [2]: [web_site_sk#82, web_site_id#83] - -(41) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [2]: [web_site_sk#82, web_site_id#83] -Arguments: [wsr_web_site_sk#60], [web_site_sk#82], Inner, BuildRight - -(42) CometProject -Input [7]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_sk#82, web_site_id#83] -Arguments: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83], [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] - -(43) CometHashAggregate -Input [5]: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] -Keys [1]: [web_site_id#83] -Functions [4]: [partial_sum(UnscaledValue(sales_price#62)), partial_sum(UnscaledValue(return_amt#64)), partial_sum(UnscaledValue(profit#63)), partial_sum(UnscaledValue(net_loss#65))] - -(44) CometExchange -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Arguments: hashpartitioning(web_site_id#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(45) ColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] - -(46) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Keys [1]: [web_site_id#83] -Functions [4]: [sum(UnscaledValue(sales_price#62)), sum(UnscaledValue(return_amt#64)), sum(UnscaledValue(profit#63)), sum(UnscaledValue(net_loss#65))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#62))#88, sum(UnscaledValue(return_amt#64))#89, sum(UnscaledValue(profit#63))#90, sum(UnscaledValue(net_loss#65))#91] -Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#83) AS id#93, MakeDecimal(sum(UnscaledValue(sales_price#62))#88,17,2) AS sales#94, MakeDecimal(sum(UnscaledValue(return_amt#64))#89,17,2) AS returns#95, (MakeDecimal(sum(UnscaledValue(profit#63))#90,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#65))#91,17,2)) AS profit#96] - -(47) Union - -(48) HashAggregate [codegen id : 4] -Input [5]: [channel#33, id#34, sales#35, returns#36, profit#37] -Keys [2]: [channel#33, id#34] -Functions [3]: [partial_sum(sales#35), partial_sum(returns#36), partial_sum(profit#37)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(49) Exchange -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#33, id#34, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(50) HashAggregate [codegen id : 5] -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#109, sum(returns#36)#110, sum(profit#37)#111] -Results [5]: [channel#33, id#34, cast(sum(sales#35)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#36)#110 as decimal(37,2)) AS returns#113, cast(sum(profit#37)#111 as decimal(38,2)) AS profit#114] - -(51) ReusedExchange [Reuses operator id: 49] -Output [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(52) HashAggregate [codegen id : 10] -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#109, sum(returns#36)#110, sum(profit#37)#111] -Results [4]: [channel#33, sum(sales#35)#109 AS sales#115, sum(returns#36)#110 AS returns#116, sum(profit#37)#111 AS profit#117] - -(53) HashAggregate [codegen id : 10] -Input [4]: [channel#33, sales#115, returns#116, profit#117] -Keys [1]: [channel#33] -Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] -Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Results [7]: [channel#33, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(54) Exchange -Input [7]: [channel#33, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Arguments: hashpartitioning(channel#33, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(55) HashAggregate [codegen id : 11] -Input [7]: [channel#33, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Keys [1]: [channel#33] -Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] -Aggregate Attributes [3]: [sum(sales#115)#130, sum(returns#116)#131, sum(profit#117)#132] -Results [5]: [channel#33, null AS id#133, sum(sales#115)#130 AS sum(sales)#134, sum(returns#116)#131 AS sum(returns)#135, sum(profit#117)#132 AS sum(profit)#136] - -(56) ReusedExchange [Reuses operator id: 49] -Output [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(57) HashAggregate [codegen id : 16] -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#109, sum(returns#36)#110, sum(profit#37)#111] -Results [3]: [sum(sales#35)#109 AS sales#137, sum(returns#36)#110 AS returns#138, sum(profit#37)#111 AS profit#139] - -(58) HashAggregate [codegen id : 16] -Input [3]: [sales#137, returns#138, profit#139] +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53], [cp_catalog_page_sk#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#52, 16, true, false, true) AS cp_catalog_page_id#53] + +(36) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53] + +(37) CometBroadcastHashJoin +Left output [5]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [page_sk#34], [cp_catalog_page_sk#51], Inner, BuildRight + +(38) CometProject +Input [7]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53], [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] + +(39) CometHashAggregate +Input [5]: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [partial_sum(UnscaledValue(sales_price#36)), partial_sum(UnscaledValue(return_amt#38)), partial_sum(UnscaledValue(profit#37)), partial_sum(UnscaledValue(net_loss#39))] + +(40) CometExchange +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(41) CometHashAggregate +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [sum(UnscaledValue(sales_price#36)), sum(UnscaledValue(return_amt#38)), sum(UnscaledValue(profit#37)), sum(UnscaledValue(net_loss#39))] + +(42) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] +PushedFilters: [IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(43) CometFilter +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_web_site_sk#58) + +(44) CometProject +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Arguments: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67], [ws_web_site_sk#58 AS wsr_web_site_sk#62, ws_sold_date_sk#61 AS date_sk#63, ws_ext_sales_price#59 AS sales_price#64, ws_net_profit#60 AS profit#65, 0.00 AS return_amt#66, 0.00 AS net_loss#67] + +(45) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#72)] +ReadSchema: struct + +(46) CometBroadcastExchange +Input [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Arguments: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] + +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(48) CometFilter +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Condition : ((isnotnull(ws_item_sk#73) AND isnotnull(ws_order_number#75)) AND isnotnull(ws_web_site_sk#74)) + +(49) CometProject +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Arguments: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75], [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] + +(50) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Right output [3]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wr_item_sk#68, wr_order_number#69], [ws_item_sk#73, ws_order_number#75], Inner, BuildLeft + +(51) CometProject +Input [8]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72, ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82], [ws_web_site_sk#74 AS wsr_web_site_sk#77, wr_returned_date_sk#72 AS date_sk#78, 0.00 AS sales_price#79, 0.00 AS profit#80, wr_return_amt#70 AS return_amt#81, wr_net_loss#71 AS net_loss#82] + +(52) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Child 1 Input [6]: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82] + +(53) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#83] + +(54) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [1]: [d_date_sk#83] +Arguments: [date_sk#63], [d_date_sk#83], Inner, BuildRight + +(55) CometProject +Input [7]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67, d_date_sk#83] +Arguments: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67], [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] + +(56) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#84, web_site_id#85] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(57) CometFilter +Input [2]: [web_site_sk#84, web_site_id#85] +Condition : isnotnull(web_site_sk#84) + +(58) CometProject +Input [2]: [web_site_sk#84, web_site_id#85] +Arguments: [web_site_sk#84, web_site_id#86], [web_site_sk#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#85, 16, true, false, true) AS web_site_id#86] + +(59) CometBroadcastExchange +Input [2]: [web_site_sk#84, web_site_id#86] +Arguments: [web_site_sk#84, web_site_id#86] + +(60) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [2]: [web_site_sk#84, web_site_id#86] +Arguments: [wsr_web_site_sk#62], [web_site_sk#84], Inner, BuildRight + +(61) CometProject +Input [7]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_sk#84, web_site_id#86] +Arguments: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86], [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] + +(62) CometHashAggregate +Input [5]: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] +Keys [1]: [web_site_id#86] +Functions [4]: [partial_sum(UnscaledValue(sales_price#64)), partial_sum(UnscaledValue(return_amt#66)), partial_sum(UnscaledValue(profit#65)), partial_sum(UnscaledValue(net_loss#67))] + +(63) CometExchange +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Arguments: hashpartitioning(web_site_id#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(64) CometHashAggregate +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Keys [1]: [web_site_id#86] +Functions [4]: [sum(UnscaledValue(sales_price#64)), sum(UnscaledValue(return_amt#66)), sum(UnscaledValue(profit#65)), sum(UnscaledValue(net_loss#67))] + +(65) CometUnion +Child 0 Input [5]: [channel#91, id#92, sales#93, returns#94, profit#95] +Child 1 Input [5]: [channel#96, id#97, sales#98, returns#99, profit#100] +Child 2 Input [5]: [channel#101, id#102, sales#103, returns#104, profit#105] + +(66) CometHashAggregate +Input [5]: [channel#91, id#92, sales#93, returns#94, profit#95] +Keys [2]: [channel#91, id#92] +Functions [3]: [partial_sum(sales#93), partial_sum(returns#94), partial_sum(profit#95)] + +(67) CometExchange +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#91, id#92, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(68) CometHashAggregate +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [2]: [channel#91, id#92] +Functions [3]: [sum(sales#93), sum(returns#94), sum(profit#95)] + +(69) ReusedExchange [Reuses operator id: 67] +Output [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(70) CometHashAggregate +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [2]: [channel#91, id#92] +Functions [3]: [sum(sales#93), sum(returns#94), sum(profit#95)] + +(71) CometHashAggregate +Input [4]: [channel#91, sales#112, returns#113, profit#114] +Keys [1]: [channel#91] +Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] + +(72) CometExchange +Input [7]: [channel#91, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] +Arguments: hashpartitioning(channel#91, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(73) CometHashAggregate +Input [7]: [channel#91, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] +Keys [1]: [channel#91] +Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] + +(74) ReusedExchange [Reuses operator id: 67] +Output [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(75) CometHashAggregate +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [2]: [channel#91, id#92] +Functions [3]: [sum(sales#93), sum(returns#94), sum(profit#95)] + +(76) CometHashAggregate +Input [3]: [sales#121, returns#122, profit#123] Keys: [] -Functions [3]: [partial_sum(sales#137), partial_sum(returns#138), partial_sum(profit#139)] -Aggregate Attributes [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Results [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] +Functions [3]: [partial_sum(sales#121), partial_sum(returns#122), partial_sum(profit#123)] -(59) Exchange -Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(77) CometExchange +Input [6]: [sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(60) HashAggregate [codegen id : 17] -Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] +(78) CometHashAggregate +Input [6]: [sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] Keys: [] -Functions [3]: [sum(sales#137), sum(returns#138), sum(profit#139)] -Aggregate Attributes [3]: [sum(sales#137)#152, sum(returns#138)#153, sum(profit#139)#154] -Results [5]: [null AS channel#155, null AS id#156, sum(sales#137)#152 AS sum(sales)#157, sum(returns#138)#153 AS sum(returns)#158, sum(profit#139)#154 AS sum(profit)#159] +Functions [3]: [sum(sales#121), sum(returns#122), sum(profit#123)] -(61) Union +(79) CometUnion +Child 0 Input [5]: [channel#91, id#92, sales#130, returns#131, profit#132] +Child 1 Input [5]: [channel#91, id#133, sum(sales)#134, sum(returns)#135, sum(profit)#136] +Child 2 Input [5]: [channel#137, id#138, sum(sales)#139, sum(returns)#140, sum(profit)#141] -(62) HashAggregate [codegen id : 18] -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Keys [5]: [channel#33, id#34, sales#112, returns#113, profit#114] +(80) CometHashAggregate +Input [5]: [channel#91, id#92, sales#130, returns#131, profit#132] +Keys [5]: [channel#91, id#92, sales#130, returns#131, profit#132] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -(63) Exchange -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Arguments: hashpartitioning(channel#33, id#34, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(81) CometExchange +Input [5]: [channel#91, id#92, sales#130, returns#131, profit#132] +Arguments: hashpartitioning(channel#91, id#92, sales#130, returns#131, profit#132, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(64) HashAggregate [codegen id : 19] -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Keys [5]: [channel#33, id#34, sales#112, returns#113, profit#114] +(82) CometHashAggregate +Input [5]: [channel#91, id#92, sales#130, returns#131, profit#132] +Keys [5]: [channel#91, id#92, sales#130, returns#131, profit#132] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -(65) TakeOrderedAndProject -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Arguments: 100, [channel#33 ASC NULLS FIRST, id#34 ASC NULLS FIRST], [channel#33, id#34, sales#112, returns#113, profit#114] +(83) CometTakeOrderedAndProject +Input [5]: [channel#91, id#92, sales#130, returns#131, profit#132] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#91 ASC NULLS FIRST,id#92 ASC NULLS FIRST], output=[channel#91,id#92,sales#130,returns#131,profit#132]), [channel#91, id#92, sales#130, returns#131, profit#132], 100, 0, [channel#91 ASC NULLS FIRST, id#92 ASC NULLS FIRST], [channel#91, id#92, sales#130, returns#131, profit#132] + +(84) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#91, id#92, sales#130, returns#131, profit#132] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/extended.txt new file mode 100644 index 0000000000..3b96b1117a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/extended.txt @@ -0,0 +1,236 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastExchange + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastExchange + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_page + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 233 out of 233 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/simplified.txt index fd65d68eea..73ebe96769 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/simplified.txt @@ -1,87 +1,86 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (19) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [cp_catalog_page_id,sum,sum,sum,sum] #3 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #6 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #7 - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - ReusedExchange [web_site_sk,web_site_id] #5 - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #8 - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #9 - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometExchange [channel,id,sales,returns,profit] #1 + CometHashAggregate [channel,id,sales,returns,profit] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id] #2 + CometHashAggregate [sales,returns,profit] [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,sum,sum,sum] [channel,id,sales,returns,profit,s_store_id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss))] + CometExchange [s_store_id] #3 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometHashAggregate [sum,sum,sum,sum] [channel,id,sales,returns,profit,cp_catalog_page_id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss))] + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometHashAggregate [sum,sum,sum,sum] [channel,id,sales,returns,profit,web_site_id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss))] + CometExchange [web_site_id] #8 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sum(sales),sum(returns),sum(profit),sum(sales),sum(returns),sum(profit)] + CometExchange [channel] #11 + CometHashAggregate [sales,returns,profit] [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sum(sales),sum(returns),sum(profit),sum(sales),sum(returns),sum(profit)] + CometExchange #12 + CometHashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/explain.txt index 58bf87acf3..0b352336e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/explain.txt @@ -1,243 +1,274 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometFilter (36) - +- CometHashAggregate (35) - +- CometExchange (34) - +- CometHashAggregate (33) - +- CometProject (32) - +- CometBroadcastHashJoin (31) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`item` (21) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer_address` +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometNativeScan parquet spark_catalog.default.item (22) + + +(1) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] -Arguments: [ca_address_sk#1, ca_state#2] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct (2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] Condition : isnotnull(ca_address_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(4) CometFilter -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(6) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#1, ca_state#2] -Right output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_address_sk#1], [c_current_addr_sk#4], Inner, BuildRight - -(7) CometProject -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_state#2, c_customer_sk#3], [ca_state#2, c_customer_sk#3] - -(8) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(9) CometFilter -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) - -(10) CometBroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(11) CometBroadcastHashJoin -Left output [2]: [ca_state#2, c_customer_sk#3] -Right output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], Inner, BuildRight - -(12) CometProject -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7], [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8, d_month_seq#9] - -(14) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(3) CometProject +Input [2]: [ca_address_sk#1, ca_state#2] +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] + +(4) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) + +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] + +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight + +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] + +(9) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) + +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] + +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight + +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] + +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) + +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] + +(20) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) + +(22) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_current_price#16, i_category#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(23) CometFilter +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true)) + +(24) CometProject +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_current_price#16, i_category#18], [i_current_price#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#18] -(15) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +(25) CometHashAggregate +Input [2]: [i_current_price#16, i_category#18] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] -(16) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +(26) CometExchange +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) CometBroadcastHashJoin -Left output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(27) CometHashAggregate +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] -(18) CometProject -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ca_state#2, ss_item_sk#5], [ca_state#2, ss_item_sk#5] +(28) CometFilter +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Arguments: [i_item_sk#12, i_current_price#13, i_category#14] +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] -(20) CometFilter -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight -(21) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_current_price#15, i_category#16] -Arguments: [i_current_price#15, i_category#16] +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] -(22) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] -(23) CometHashAggregate -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight -(24) CometExchange -Input [3]: [i_category#16, sum#17, count#18] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] -(25) CometHashAggregate -Input [3]: [i_category#16, sum#17, count#18] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] - -(26) CometFilter -Input [2]: [avg(i_current_price)#19, i_category#16] -Condition : isnotnull(avg(i_current_price)#19) - -(27) CometBroadcastExchange -Input [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [avg(i_current_price)#19, i_category#16] - -(28) CometBroadcastHashJoin -Left output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Right output [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [i_category#14], [i_category#16], Inner, (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight - -(29) CometProject -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#19, i_category#16] -Arguments: [i_item_sk#12], [i_item_sk#12] - -(30) CometBroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: [i_item_sk#12] - -(31) CometBroadcastHashJoin -Left output [2]: [ca_state#2, ss_item_sk#5] -Right output [1]: [i_item_sk#12] -Arguments: [ss_item_sk#5], [i_item_sk#12], Inner, BuildRight - -(32) CometProject -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] -Arguments: [ca_state#2], [ca_state#2] - -(33) CometHashAggregate -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] -(34) CometExchange -Input [2]: [ca_state#2, count#20] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(36) CometExchange +Input [2]: [ca_state#3, count#22] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) CometHashAggregate -Input [2]: [ca_state#2, count#20] -Keys [1]: [ca_state#2] +(37) CometHashAggregate +Input [2]: [ca_state#3, count#22] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -(36) CometFilter -Input [3]: [state#21, cnt#22, ca_state#2] -Condition : (cnt#22 >= 10) +(38) CometFilter +Input [3]: [state#23, cnt#24, ca_state#3] +Condition : (cnt#24 >= 10) -(37) CometTakeOrderedAndProject -Input [3]: [state#21, cnt#22, ca_state#2] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST,ca_state#2 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, [cnt#22 ASC NULLS FIRST, ca_state#2 ASC NULLS FIRST], [state#21, cnt#22] +(39) CometTakeOrderedAndProject +Input [3]: [state#23, cnt#24, ca_state#3] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#23, cnt#24] -(38) ColumnarToRow [codegen id : 1] -Input [2]: [state#21, cnt#22] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (45) -+- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometFilter (40) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23, d_year#24, d_moy#25] +(41) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] +ReadSchema: struct -(40) CometFilter -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(41) CometProject -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23], [d_month_seq#23] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(42) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(43) CometExchange -Input [1]: [d_month_seq#23] -Arguments: hashpartitioning(d_month_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(44) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(45) ColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#23] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/extended.txt new file mode 100644 index 0000000000..f6467c6fd9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/extended.txt @@ -0,0 +1,50 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 45 out of 46 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/simplified.txt index 305e08545d..6f55ccc7cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [state,cnt,ca_state] CometFilter [state,cnt,ca_state] - CometHashAggregate [state,cnt,ca_state,count,count(1)] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] @@ -14,37 +14,39 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometProject [ca_state,c_customer_sk] CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [d_month_seq] CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] - CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] CometExchange [i_category] #8 - CometHashAggregate [i_category,sum,count,i_current_price] - CometFilter [i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_current_price,i_category] + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/explain.txt index d0b3132fc7..3ad67801f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/explain.txt @@ -1,41 +1,41 @@ == Physical Plan == -* ColumnarToRow (163) -+- CometSort (162) - +- CometColumnarExchange (161) - +- CometProject (160) - +- CometSortMergeJoin (159) - :- CometSort (98) - : +- CometExchange (97) - : +- CometHashAggregate (96) - : +- CometHashAggregate (95) - : +- CometProject (94) - : +- CometBroadcastHashJoin (93) - : :- CometProject (88) - : : +- CometBroadcastHashJoin (87) - : : :- CometProject (85) - : : : +- CometBroadcastHashJoin (84) - : : : :- CometProject (82) - : : : : +- CometBroadcastHashJoin (81) - : : : : :- CometProject (79) - : : : : : +- CometBroadcastHashJoin (78) - : : : : : :- CometProject (74) - : : : : : : +- CometBroadcastHashJoin (73) - : : : : : : :- CometProject (71) - : : : : : : : +- CometBroadcastHashJoin (70) - : : : : : : : :- CometProject (66) - : : : : : : : : +- CometBroadcastHashJoin (65) - : : : : : : : : :- CometProject (61) - : : : : : : : : : +- CometBroadcastHashJoin (60) - : : : : : : : : : :- CometProject (58) - : : : : : : : : : : +- CometBroadcastHashJoin (57) - : : : : : : : : : : :- CometProject (53) - : : : : : : : : : : : +- CometBroadcastHashJoin (52) - : : : : : : : : : : : :- CometProject (50) - : : : : : : : : : : : : +- CometBroadcastHashJoin (49) - : : : : : : : : : : : : :- CometProject (45) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (44) - : : : : : : : : : : : : : :- CometProject (40) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (39) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) : : : : : : : : : : : : : : :- CometProject (35) : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) : : : : : : : : : : : : : : : :- CometProject (30) @@ -46,10 +46,10 @@ : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,111 +62,120 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (11) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (31) - : : : : : : : : : : : : : : +- CometBroadcastExchange (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (36) - : : : : : : : : : : : : : +- CometBroadcastExchange (43) - : : : : : : : : : : : : : +- CometFilter (42) - : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (41) - : : : : : : : : : : : : +- CometBroadcastExchange (48) - : : : : : : : : : : : : +- CometFilter (47) - : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (46) - : : : : : : : : : : : +- ReusedExchange (51) - : : : : : : : : : : +- CometBroadcastExchange (56) - : : : : : : : : : : +- CometFilter (55) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (54) - : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : +- CometBroadcastExchange (64) - : : : : : : : : +- CometFilter (63) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (62) - : : : : : : : +- CometBroadcastExchange (69) - : : : : : : : +- CometFilter (68) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (67) - : : : : : : +- ReusedExchange (72) - : : : : : +- CometBroadcastExchange (77) - : : : : : +- CometFilter (76) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (75) - : : : : +- ReusedExchange (80) - : : : +- ReusedExchange (83) - : : +- ReusedExchange (86) - : +- CometBroadcastExchange (92) - : +- CometProject (91) - : +- CometFilter (90) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (89) - +- CometSort (158) - +- CometExchange (157) - +- CometHashAggregate (156) - +- CometHashAggregate (155) - +- CometProject (154) - +- CometBroadcastHashJoin (153) - :- CometProject (151) - : +- CometBroadcastHashJoin (150) - : :- CometProject (148) - : : +- CometBroadcastHashJoin (147) - : : :- CometProject (145) - : : : +- CometBroadcastHashJoin (144) - : : : :- CometProject (142) - : : : : +- CometBroadcastHashJoin (141) - : : : : :- CometProject (139) - : : : : : +- CometBroadcastHashJoin (138) - : : : : : :- CometProject (136) - : : : : : : +- CometBroadcastHashJoin (135) - : : : : : : :- CometProject (133) - : : : : : : : +- CometBroadcastHashJoin (132) - : : : : : : : :- CometProject (130) - : : : : : : : : +- CometBroadcastHashJoin (129) - : : : : : : : : :- CometProject (127) - : : : : : : : : : +- CometBroadcastHashJoin (126) - : : : : : : : : : :- CometProject (124) - : : : : : : : : : : +- CometBroadcastHashJoin (123) - : : : : : : : : : : :- CometProject (121) - : : : : : : : : : : : +- CometBroadcastHashJoin (120) - : : : : : : : : : : : :- CometProject (118) - : : : : : : : : : : : : +- CometBroadcastHashJoin (117) - : : : : : : : : : : : : :- CometProject (115) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (114) - : : : : : : : : : : : : : :- CometProject (112) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (111) - : : : : : : : : : : : : : : :- CometProject (107) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (106) - : : : : : : : : : : : : : : : :- CometSort (100) - : : : : : : : : : : : : : : : : +- ReusedExchange (99) - : : : : : : : : : : : : : : : +- CometSort (105) - : : : : : : : : : : : : : : : +- CometProject (104) - : : : : : : : : : : : : : : : +- CometFilter (103) - : : : : : : : : : : : : : : : +- CometHashAggregate (102) - : : : : : : : : : : : : : : : +- ReusedExchange (101) - : : : : : : : : : : : : : : +- CometBroadcastExchange (110) - : : : : : : : : : : : : : : +- CometFilter (109) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (108) - : : : : : : : : : : : : : +- ReusedExchange (113) - : : : : : : : : : : : : +- ReusedExchange (116) - : : : : : : : : : : : +- ReusedExchange (119) - : : : : : : : : : : +- ReusedExchange (122) - : : : : : : : : : +- ReusedExchange (125) - : : : : : : : : +- ReusedExchange (128) - : : : : : : : +- ReusedExchange (131) - : : : : : : +- ReusedExchange (134) - : : : : : +- ReusedExchange (137) - : : : : +- ReusedExchange (140) - : : : +- ReusedExchange (143) - : : +- ReusedExchange (146) - : +- ReusedExchange (149) - +- ReusedExchange (152) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometNativeScan parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] +ReadSchema: struct (2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -176,9 +185,12 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (5) CometFilter Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] @@ -205,9 +217,12 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` +(11) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct (12) CometFilter Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] @@ -225,9 +240,12 @@ Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct (17) CometFilter Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] @@ -289,9 +307,12 @@ Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(31) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#31, d_year#32] -Arguments: [d_date_sk#31, d_year#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct (32) CometFilter Input [2]: [d_date_sk#31, d_year#32] @@ -310,528 +331,579 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(36) CometNativeScan: `spark_catalog`.`default`.`store` +(36) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] +ReadSchema: struct (37) CometFilter Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(s_zip#35)) +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(38) CometBroadcastExchange +(38) CometProject Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] + +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] -(39) CometBroadcastHashJoin +(40) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(40) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] - -(41) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(42) CometFilter -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Condition : (((((isnotnull(c_customer_sk#36) AND isnotnull(c_first_sales_date_sk#41)) AND isnotnull(c_first_shipto_date_sk#40)) AND isnotnull(c_current_cdemo_sk#37)) AND isnotnull(c_current_hdemo_sk#38)) AND isnotnull(c_current_addr_sk#39)) - -(43) CometBroadcastExchange -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(44) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] -Right output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_customer_sk#2], [c_customer_sk#36], Inner, BuildRight - -(45) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(46) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(47) CometFilter -Input [2]: [d_date_sk#42, d_year#43] -Condition : isnotnull(d_date_sk#42) - -(48) CometBroadcastExchange -Input [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(49) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Right output [2]: [d_date_sk#42, d_year#43] -Arguments: [c_first_sales_date_sk#41], [d_date_sk#42], Inner, BuildRight - -(50) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41, d_date_sk#42, d_year#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] - -(51) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#44, d_year#45] - -(52) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_shipto_date_sk#40], [d_date_sk#44], Inner, BuildRight - -(53) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(54) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(55) CometFilter -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Condition : (isnotnull(cd_demo_sk#46) AND isnotnull(cd_marital_status#47)) - -(56) CometBroadcastExchange -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(57) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#46], Inner, BuildRight - -(58) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] - -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#48, cd_marital_status#49] - -(60) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [c_current_cdemo_sk#37], [cd_demo_sk#48], Inner, NOT (cd_marital_status#47 = cd_marital_status#49), BuildRight - -(61) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47, cd_demo_sk#48, cd_marital_status#49] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(62) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(63) CometFilter -Input [1]: [p_promo_sk#50] -Condition : isnotnull(p_promo_sk#50) - -(64) CometBroadcastExchange -Input [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(65) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [1]: [p_promo_sk#50] -Arguments: [ss_promo_sk#7], [p_promo_sk#50], Inner, BuildRight - -(66) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, p_promo_sk#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(67) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] - -(68) CometFilter -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Condition : (isnotnull(hd_demo_sk#51) AND isnotnull(hd_income_band_sk#52)) - -(69) CometBroadcastExchange -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] + +(42) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) + +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] + +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight + +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] + +(47) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) + +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] + +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight + +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] + +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] + +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight + +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] + +(55) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) + +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] + +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] + +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight + +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] + +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] + +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] + +(64) CometNativeScan parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) + +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] + +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight + +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] + +(69) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct -(70) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#51], Inner, BuildRight +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(71) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(72) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] - -(73) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [c_current_hdemo_sk#38], [hd_demo_sk#53], Inner, BuildRight +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight + +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(74) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(75) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(76) CometFilter -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Condition : isnotnull(ca_address_sk#55) +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(77) CometBroadcastExchange -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +(77) CometNativeScan parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(78) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] -Right output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_addr_sk#5], [ca_address_sk#55], Inner, BuildRight +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(80) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Right output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [c_current_addr_sk#39], [ca_address_sk#60], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(83) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#65] +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (84) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#65] -Arguments: [hd_income_band_sk#52], [ib_income_band_sk#65], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight (85) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#65] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(86) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#66] - -(87) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#66] -Arguments: [hd_income_band_sk#54], [ib_income_band_sk#66], Inner, BuildRight - -(88) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#66] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(89) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] - -(90) CometFilter -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Condition : ((((((isnotnull(i_current_price#68) AND i_color#69 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#68 >= 64.00)) AND (i_current_price#68 <= 74.00)) AND (i_current_price#68 >= 65.00)) AND (i_current_price#68 <= 79.00)) AND isnotnull(i_item_sk#67)) - -(91) CometProject -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70], [i_item_sk#67, i_product_name#70] - -(92) CometBroadcastExchange -Input [2]: [i_item_sk#67, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70] - -(93) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [2]: [i_item_sk#67, i_product_name#70] -Arguments: [ss_item_sk#1], [i_item_sk#67], Inner, BuildRight - -(94) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] - -(95) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(86) CometNativeScan parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) + +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] + +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight + +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] + +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight + +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(94) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) + +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] + +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] + +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight + +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] + +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -(96) CometHashAggregate -Input [19]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45, count#71, sum#72, sum#73, sum#74] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(97) CometExchange -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: hashpartitioning(item_sk#76, store_name#77, store_zip#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(98) CometSort -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91], [item_sk#76 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, store_zip#78 ASC NULLS FIRST] - -(99) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(100) CometSort -Input [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92 ASC NULLS FIRST] +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(101) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] - -(102) CometHashAggregate -Input [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] -Keys [1]: [cs_item_sk#103] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#107)), sum(((cr_refunded_cash#108 + cr_reversed_charge#109) + cr_store_credit#110))] - -(103) CometFilter -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) - -(104) CometProject -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Arguments: [cs_item_sk#103], [cs_item_sk#103] +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] (105) CometSort -Input [1]: [cs_item_sk#103] -Arguments: [cs_item_sk#103], [cs_item_sk#103 ASC NULLS FIRST] +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(106) CometSortMergeJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [1]: [cs_item_sk#103] -Arguments: [ss_item_sk#92], [cs_item_sk#103], Inner +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(107) CometProject -Input [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, cs_item_sk#103] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] -(108) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(109) CometFilter -Input [2]: [d_date_sk#111, d_year#112] -Condition : ((isnotnull(d_year#112) AND (d_year#112 = 2000)) AND isnotnull(d_date_sk#111)) +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(110) CometBroadcastExchange -Input [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(111) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ss_sold_date_sk#102], [d_date_sk#111], Inner, BuildRight +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner (112) CometProject -Input [13]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, d_date_sk#111, d_year#112] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] - -(113) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] - -(114) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] -Right output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_store_sk#97], [s_store_sk#113], Inner, BuildRight - -(115) CometProject -Input [14]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] - -(116) ReusedExchange [Reuses operator id: 43] -Output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(117) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] -Right output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_customer_sk#93], [c_customer_sk#116], Inner, BuildRight +(113) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) + +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] + +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight + +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] + +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] + +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight + +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] + +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight + +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] + +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight + +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] + +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] + +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight + +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] + +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight + +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] + +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] + +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] -(118) CometProject -Input [18]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight -(119) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#122, d_year#123] +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(120) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [c_first_sales_date_sk#121], [d_date_sk#122], Inner, BuildRight - -(121) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] - -(122) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#124, d_year#125] - -(123) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [c_first_shipto_date_sk#120], [d_date_sk#124], Inner, BuildRight - -(124) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123, d_date_sk#124, d_year#125] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(125) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#126, cd_marital_status#127] - -(126) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_cdemo_sk#94], [cd_demo_sk#126], Inner, BuildRight - -(127) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] - -(128) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#128, cd_marital_status#129] - -(129) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] -Right output [2]: [cd_demo_sk#128, cd_marital_status#129] -Arguments: [c_current_cdemo_sk#117], [cd_demo_sk#128], Inner, NOT (cd_marital_status#127 = cd_marital_status#129), BuildRight - -(130) CometProject -Input [18]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127, cd_demo_sk#128, cd_marital_status#129] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(131) ReusedExchange [Reuses operator id: 64] -Output [1]: [p_promo_sk#130] - -(132) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [1]: [p_promo_sk#130] -Arguments: [ss_promo_sk#98], [p_promo_sk#130], Inner, BuildRight - -(133) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, p_promo_sk#130] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(134) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#131, hd_income_band_sk#132] +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -(135) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_hdemo_sk#95], [hd_demo_sk#131], Inner, BuildRight +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight -(136) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(137) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] -(138) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] -Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [c_current_hdemo_sk#118], [hd_demo_sk#133], Inner, BuildRight +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight -(139) CometProject -Input [15]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] -(140) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(141) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] -Right output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_addr_sk#96], [ca_address_sk#135], Inner, BuildRight +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight -(142) CometProject -Input [18]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(143) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(144) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Right output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [c_current_addr_sk#119], [ca_address_sk#140], Inner, BuildRight +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight -(145) CometProject -Input [21]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(146) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#145] +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] -(147) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#145] -Arguments: [hd_income_band_sk#132], [ib_income_band_sk#145], Inner, BuildRight +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight -(148) CometProject -Input [20]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#145] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(149) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#146] +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] -(150) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#146] -Arguments: [hd_income_band_sk#134], [ib_income_band_sk#146], Inner, BuildRight +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight -(151) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#146] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(152) ReusedExchange [Reuses operator id: 92] -Output [2]: [i_item_sk#147, i_product_name#148] +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] -(153) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [2]: [i_item_sk#147, i_product_name#148] -Arguments: [ss_item_sk#92], [i_item_sk#147], Inner, BuildRight +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight -(154) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Arguments: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148], [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] -(155) CometHashAggregate -Input [18]: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#99)), partial_sum(UnscaledValue(ss_list_price#100)), partial_sum(UnscaledValue(ss_coupon_amt#101))] +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] -(156) CometHashAggregate -Input [19]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125, count#71, sum#149, sum#150, sum#151] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#99)), sum(UnscaledValue(ss_list_price#100)), sum(UnscaledValue(ss_coupon_amt#101))] +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] -(157) CometExchange -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: hashpartitioning(item_sk#152, store_name#153, store_zip#154, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(158) CometSort -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159], [item_sk#152 ASC NULLS FIRST, store_name#153 ASC NULLS FIRST, store_zip#154 ASC NULLS FIRST] +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] -(159) CometSortMergeJoin -Left output [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Right output [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#76, store_name#77, store_zip#78], [item_sk#152, store_name#153, store_zip#154], Inner, (cnt#156 <= cnt#88) +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) -(160) CometProject -Input [25]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] -(161) CometColumnarExchange -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: rangepartitioning(product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, s1#89 ASC NULLS FIRST, s1#157 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(162) CometSort -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, s1#89 ASC NULLS FIRST, s1#157 ASC NULLS FIRST] +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST] -(163) ColumnarToRow [codegen id : 1] -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/extended.txt new file mode 100644 index 0000000000..ae10068c4c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/extended.txt @@ -0,0 +1,237 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometProject + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.income_band + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 234 out of 234 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/simplified.txt index c68b3e5cbd..cda6c1a5bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt,s1,s1] #1 + CometExchange [product_name,store_name,cnt,s1,s1] #1 CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #2 - CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -47,66 +47,71 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] CometExchange [cs_item_sk] #5 - CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #11 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #12 CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - CometBroadcastExchange [i_item_sk,i_product_name] #16 - CometProject [i_item_sk,i_product_name] + CometBroadcastExchange [ib_income_band_sk] #16 + CometFilter [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 + CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #17 - CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometExchange [item_sk,store_name,store_zip] #18 + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -144,11 +149,11 @@ WholeStageCodegen (1) CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - CometBroadcastExchange [d_date_sk,d_year] #18 + CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #9 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 ReusedExchange [d_date_sk,d_year] #11 @@ -160,6 +165,6 @@ WholeStageCodegen (1) ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [i_item_sk,i_product_name] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/explain.txt index bd7991a8ef..43ec5f0ac0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/explain.txt @@ -1,88 +1,98 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- Window (69) - +- WindowGroupLimit (68) - +- * Sort (67) - +- Exchange (66) - +- WindowGroupLimit (65) - +- * ColumnarToRow (64) - +- CometSort (63) - +- CometUnion (62) - :- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (14) - :- CometHashAggregate (26) - : +- CometExchange (25) - : +- CometHashAggregate (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - :- CometHashAggregate (31) - : +- CometExchange (30) - : +- CometHashAggregate (29) - : +- CometHashAggregate (28) - : +- ReusedExchange (27) - :- CometHashAggregate (36) - : +- CometExchange (35) - : +- CometHashAggregate (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- ReusedExchange (37) - :- CometHashAggregate (46) - : +- CometExchange (45) - : +- CometHashAggregate (44) - : +- CometHashAggregate (43) - : +- ReusedExchange (42) - :- CometHashAggregate (51) - : +- CometExchange (50) - : +- CometHashAggregate (49) - : +- CometHashAggregate (48) - : +- ReusedExchange (47) - :- CometHashAggregate (56) - : +- CometExchange (55) - : +- CometHashAggregate (54) - : +- CometHashAggregate (53) - : +- ReusedExchange (52) - +- CometHashAggregate (61) - +- CometExchange (60) - +- CometHashAggregate (59) - +- CometHashAggregate (58) - +- ReusedExchange (57) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (74) ++- * Filter (73) + +- Window (72) + +- WindowGroupLimit (71) + +- * CometColumnarToRow (70) + +- CometSort (69) + +- CometColumnarExchange (68) + +- WindowGroupLimit (67) + +- * CometColumnarToRow (66) + +- CometSort (65) + +- CometUnion (64) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + :- CometHashAggregate (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- CometHashAggregate (30) + : +- ReusedExchange (29) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometHashAggregate (35) + : +- ReusedExchange (34) + :- CometHashAggregate (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometHashAggregate (40) + : +- ReusedExchange (39) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometHashAggregate (45) + : +- ReusedExchange (44) + :- CometHashAggregate (53) + : +- CometExchange (52) + : +- CometHashAggregate (51) + : +- CometHashAggregate (50) + : +- ReusedExchange (49) + :- CometHashAggregate (58) + : +- CometExchange (57) + : +- CometHashAggregate (56) + : +- CometHashAggregate (55) + : +- ReusedExchange (54) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometHashAggregate (60) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] @@ -105,281 +115,298 @@ Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] +Right output [2]: [s_store_sk#11, s_store_id#13] Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] - -(14) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) +(15) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(19) CometHashAggregate -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(20) CometExchange -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#18, isEmpty#19] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] (21) CometHashAggregate -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#18, isEmpty#19] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(22) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27, sum#28, isEmpty#29] +(22) CometExchange +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [10]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27, sum#28, isEmpty#29] -Keys [8]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27] -Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(24) CometHashAggregate -Input [8]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sumsales#32] -Keys [7]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26] -Functions [1]: [partial_sum(sumsales#32)] +(24) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] -(25) CometExchange -Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sum#33, isEmpty#34] -Arguments: hashpartitioning(i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(25) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] (26) CometHashAggregate -Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sum#33, isEmpty#34] -Keys [7]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26] -Functions [1]: [sum(sumsales#32)] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sumsales#32] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [partial_sum(sumsales#32)] -(27) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42, sum#43, isEmpty#44] +(27) CometExchange +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [10]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42, sum#43, isEmpty#44] -Keys [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42] -Functions [1]: [sum(coalesce((ss_sales_price#45 * cast(ss_quantity#46 as decimal(10,0))), 0.00))] +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [sum(sumsales#32)] -(29) CometHashAggregate -Input [7]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sumsales#47] -Keys [6]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40] -Functions [1]: [partial_sum(sumsales#47)] +(29) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] -(30) CometExchange -Input [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(30) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#40 * cast(ss_quantity#41 as decimal(10,0))), 0.00))] (31) CometHashAggregate -Input [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sum#48, isEmpty#49] -Keys [6]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40] -Functions [1]: [sum(sumsales#47)] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sumsales#42] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [partial_sum(sumsales#42)] -(32) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57, sum#58, isEmpty#59] +(32) CometExchange +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#43, isEmpty#44] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate -Input [10]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57, sum#58, isEmpty#59] -Keys [8]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57] -Functions [1]: [sum(coalesce((ss_sales_price#60 * cast(ss_quantity#61 as decimal(10,0))), 0.00))] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#43, isEmpty#44] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [sum(sumsales#42)] -(34) CometHashAggregate -Input [6]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sumsales#62] -Keys [5]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54] -Functions [1]: [partial_sum(sumsales#62)] +(34) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13, sum#48, isEmpty#49] -(35) CometExchange -Input [7]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sum#63, isEmpty#64] -Arguments: hashpartitioning(i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(35) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13, sum#48, isEmpty#49] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#50 * cast(ss_quantity#51 as decimal(10,0))), 0.00))] (36) CometHashAggregate -Input [7]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sum#63, isEmpty#64] -Keys [5]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54] -Functions [1]: [sum(sumsales#62)] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sumsales#52] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45] +Functions [1]: [partial_sum(sumsales#52)] -(37) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72, sum#73, isEmpty#74] +(37) CometExchange +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sum#53, isEmpty#54] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (38) CometHashAggregate -Input [10]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72, sum#73, isEmpty#74] -Keys [8]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72] -Functions [1]: [sum(coalesce((ss_sales_price#75 * cast(ss_quantity#76 as decimal(10,0))), 0.00))] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sum#53, isEmpty#54] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45] +Functions [1]: [sum(sumsales#52)] -(39) CometHashAggregate -Input [5]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sumsales#77] -Keys [4]: [i_category#65, i_class#66, i_brand#67, i_product_name#68] -Functions [1]: [partial_sum(sumsales#77)] +(39) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13, sum#58, isEmpty#59] -(40) CometExchange -Input [6]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sum#78, isEmpty#79] -Arguments: hashpartitioning(i_category#65, i_class#66, i_brand#67, i_product_name#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(40) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13, sum#58, isEmpty#59] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#60 * cast(ss_quantity#61 as decimal(10,0))), 0.00))] (41) CometHashAggregate -Input [6]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sum#78, isEmpty#79] -Keys [4]: [i_category#65, i_class#66, i_brand#67, i_product_name#68] -Functions [1]: [sum(sumsales#77)] +Input [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sumsales#62] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [partial_sum(sumsales#62)] -(42) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87, sum#88, isEmpty#89] +(42) CometExchange +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#63, isEmpty#64] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (43) CometHashAggregate -Input [10]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87, sum#88, isEmpty#89] -Keys [8]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87] -Functions [1]: [sum(coalesce((ss_sales_price#90 * cast(ss_quantity#91 as decimal(10,0))), 0.00))] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#63, isEmpty#64] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [sum(sumsales#62)] -(44) CometHashAggregate -Input [4]: [i_category#80, i_class#81, i_brand#82, sumsales#92] -Keys [3]: [i_category#80, i_class#81, i_brand#82] -Functions [1]: [partial_sum(sumsales#92)] +(44) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13, sum#68, isEmpty#69] -(45) CometExchange -Input [5]: [i_category#80, i_class#81, i_brand#82, sum#93, isEmpty#94] -Arguments: hashpartitioning(i_category#80, i_class#81, i_brand#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(45) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13, sum#68, isEmpty#69] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#70 * cast(ss_quantity#71 as decimal(10,0))), 0.00))] (46) CometHashAggregate -Input [5]: [i_category#80, i_class#81, i_brand#82, sum#93, isEmpty#94] -Keys [3]: [i_category#80, i_class#81, i_brand#82] -Functions [1]: [sum(sumsales#92)] +Input [4]: [i_category#21, i_class#20, i_brand#19, sumsales#72] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [partial_sum(sumsales#72)] -(47) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102, sum#103, isEmpty#104] +(47) CometExchange +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#73, isEmpty#74] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (48) CometHashAggregate -Input [10]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102, sum#103, isEmpty#104] -Keys [8]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102] -Functions [1]: [sum(coalesce((ss_sales_price#105 * cast(ss_quantity#106 as decimal(10,0))), 0.00))] +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#73, isEmpty#74] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [sum(sumsales#72)] -(49) CometHashAggregate -Input [3]: [i_category#95, i_class#96, sumsales#107] -Keys [2]: [i_category#95, i_class#96] -Functions [1]: [partial_sum(sumsales#107)] +(49) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13, sum#78, isEmpty#79] -(50) CometExchange -Input [4]: [i_category#95, i_class#96, sum#108, isEmpty#109] -Arguments: hashpartitioning(i_category#95, i_class#96, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(50) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13, sum#78, isEmpty#79] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#80 * cast(ss_quantity#81 as decimal(10,0))), 0.00))] (51) CometHashAggregate -Input [4]: [i_category#95, i_class#96, sum#108, isEmpty#109] -Keys [2]: [i_category#95, i_class#96] -Functions [1]: [sum(sumsales#107)] +Input [3]: [i_category#21, i_class#20, sumsales#82] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [partial_sum(sumsales#82)] -(52) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117, sum#118, isEmpty#119] +(52) CometExchange +Input [4]: [i_category#21, i_class#20, sum#83, isEmpty#84] +Arguments: hashpartitioning(i_category#21, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (53) CometHashAggregate -Input [10]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117, sum#118, isEmpty#119] -Keys [8]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117] -Functions [1]: [sum(coalesce((ss_sales_price#120 * cast(ss_quantity#121 as decimal(10,0))), 0.00))] +Input [4]: [i_category#21, i_class#20, sum#83, isEmpty#84] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [sum(sumsales#82)] -(54) CometHashAggregate -Input [2]: [i_category#110, sumsales#122] -Keys [1]: [i_category#110] -Functions [1]: [partial_sum(sumsales#122)] +(54) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13, sum#88, isEmpty#89] -(55) CometExchange -Input [3]: [i_category#110, sum#123, isEmpty#124] -Arguments: hashpartitioning(i_category#110, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(55) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13, sum#88, isEmpty#89] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#90 * cast(ss_quantity#91 as decimal(10,0))), 0.00))] (56) CometHashAggregate -Input [3]: [i_category#110, sum#123, isEmpty#124] -Keys [1]: [i_category#110] -Functions [1]: [sum(sumsales#122)] +Input [2]: [i_category#21, sumsales#92] +Keys [1]: [i_category#21] +Functions [1]: [partial_sum(sumsales#92)] -(57) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sum#133, isEmpty#134] +(57) CometExchange +Input [3]: [i_category#21, sum#93, isEmpty#94] +Arguments: hashpartitioning(i_category#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (58) CometHashAggregate -Input [10]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sum#133, isEmpty#134] -Keys [8]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132] -Functions [1]: [sum(coalesce((ss_sales_price#135 * cast(ss_quantity#136 as decimal(10,0))), 0.00))] +Input [3]: [i_category#21, sum#93, isEmpty#94] +Keys [1]: [i_category#21] +Functions [1]: [sum(sumsales#92)] + +(59) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13, sum#98, isEmpty#99] -(59) CometHashAggregate -Input [1]: [sumsales#137] +(60) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13, sum#98, isEmpty#99] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#100 * cast(ss_quantity#101 as decimal(10,0))), 0.00))] + +(61) CometHashAggregate +Input [1]: [sumsales#102] Keys: [] -Functions [1]: [partial_sum(sumsales#137)] +Functions [1]: [partial_sum(sumsales#102)] -(60) CometExchange -Input [2]: [sum#138, isEmpty#139] +(62) CometExchange +Input [2]: [sum#103, isEmpty#104] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(61) CometHashAggregate -Input [2]: [sum#138, isEmpty#139] +(63) CometHashAggregate +Input [2]: [sum#103, isEmpty#104] Keys: [] -Functions [1]: [sum(sumsales#137)] - -(62) CometUnion -Child 0 Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Child 1 Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#141, sumsales#142] -Child 2 Input [9]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#143, s_store_id#144, sumsales#145] -Child 3 Input [9]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#146, d_moy#147, s_store_id#148, sumsales#149] -Child 4 Input [9]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Child 5 Input [9]: [i_category#80, i_class#81, i_brand#82, i_product_name#155, d_year#156, d_qoy#157, d_moy#158, s_store_id#159, sumsales#160] -Child 6 Input [9]: [i_category#95, i_class#96, i_brand#161, i_product_name#162, d_year#163, d_qoy#164, d_moy#165, s_store_id#166, sumsales#167] -Child 7 Input [9]: [i_category#110, i_class#168, i_brand#169, i_product_name#170, d_year#171, d_qoy#172, d_moy#173, s_store_id#174, sumsales#175] -Child 8 Input [9]: [i_category#176, i_class#177, i_brand#178, i_product_name#179, d_year#180, d_qoy#181, d_moy#182, s_store_id#183, sumsales#184] - -(63) CometSort -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140], [i_category#16 ASC NULLS FIRST, sumsales#140 DESC NULLS LAST] - -(64) ColumnarToRow [codegen id : 1] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] - -(65) WindowGroupLimit -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16], [sumsales#140 DESC NULLS LAST], rank(sumsales#140), 100, Partial - -(66) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) Sort [codegen id : 2] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16 ASC NULLS FIRST, sumsales#140 DESC NULLS LAST], false, 0 - -(68) WindowGroupLimit -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16], [sumsales#140 DESC NULLS LAST], rank(sumsales#140), 100, Final - -(69) Window -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [rank(sumsales#140) windowspecdefinition(i_category#16, sumsales#140 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#185], [i_category#16], [sumsales#140 DESC NULLS LAST] - -(70) Filter [codegen id : 3] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] -Condition : (rk#185 <= 100) - -(71) TakeOrderedAndProject -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] -Arguments: 100, [i_category#16 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, d_moy#9 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sumsales#140 ASC NULLS FIRST, rk#185 ASC NULLS FIRST], [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] +Functions [1]: [sum(sumsales#102)] + +(64) CometUnion +Child 0 Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Child 1 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#114, sumsales#115] +Child 2 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#116, s_store_id#117, sumsales#118] +Child 3 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#119, d_moy#120, s_store_id#121, sumsales#122] +Child 4 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#123, d_qoy#124, d_moy#125, s_store_id#126, sumsales#127] +Child 5 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sumsales#133] +Child 6 Input [9]: [i_category#21, i_class#20, i_brand#134, i_product_name#135, d_year#136, d_qoy#137, d_moy#138, s_store_id#139, sumsales#140] +Child 7 Input [9]: [i_category#21, i_class#141, i_brand#142, i_product_name#143, d_year#144, d_qoy#145, d_moy#146, s_store_id#147, sumsales#148] +Child 8 Input [9]: [i_category#149, i_class#150, i_brand#151, i_product_name#152, d_year#153, d_qoy#154, d_moy#155, s_store_id#156, sumsales#157] + +(65) CometSort +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113], [i_category#105 ASC NULLS FIRST, sumsales#113 DESC NULLS LAST] + +(66) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] + +(67) WindowGroupLimit +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105], [sumsales#113 DESC NULLS LAST], rank(sumsales#113), 100, Partial + +(68) CometColumnarExchange +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: hashpartitioning(i_category#105, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(69) CometSort +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113], [i_category#105 ASC NULLS FIRST, sumsales#113 DESC NULLS LAST] + +(70) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] + +(71) WindowGroupLimit +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105], [sumsales#113 DESC NULLS LAST], rank(sumsales#113), 100, Final + +(72) Window +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [rank(sumsales#113) windowspecdefinition(i_category#105, sumsales#113 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#158], [i_category#105], [sumsales#113 DESC NULLS LAST] + +(73) Filter [codegen id : 3] +Input [10]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] +Condition : (rk#158 <= 100) + +(74) TakeOrderedAndProject +Input [10]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] +Arguments: 100, [i_category#105 ASC NULLS FIRST, i_class#106 ASC NULLS FIRST, i_brand#107 ASC NULLS FIRST, i_product_name#108 ASC NULLS FIRST, d_year#109 ASC NULLS FIRST, d_qoy#110 ASC NULLS FIRST, d_moy#111 ASC NULLS FIRST, s_store_id#112 ASC NULLS FIRST, sumsales#113 ASC NULLS FIRST, rk#158 ASC NULLS FIRST], [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/extended.txt new file mode 100644 index 0000000000..b9c702d0ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/extended.txt @@ -0,0 +1,244 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 235 out of 240 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/simplified.txt index 6bf8ad0746..45089e6274 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/simplified.txt @@ -5,73 +5,76 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (2) - Sort [i_category,sumsales] + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name] #9 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sumsales,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand] #10 - CometHashAggregate [i_category,i_class,i_brand,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,sumsales,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class] #11 - CometHashAggregate [i_category,i_class,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,sumsales,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category] #12 - CometHashAggregate [i_category,sum,isEmpty,sumsales] - CometHashAggregate [i_category,sumsales,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange #13 - CometHashAggregate [sum,isEmpty,sumsales] - CometHashAggregate [sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] + CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] + CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] + CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name] #9 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] + CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand] #10 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] + CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class] #11 + CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] + CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category] #12 + CometHashAggregate [sumsales] [i_category,sum,isEmpty] + CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange #13 + CometHashAggregate [sumsales] [sum,isEmpty] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/explain.txt index c29a4118cd..aaae956f64 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/explain.txt @@ -1,75 +1,90 @@ == Physical Plan == -TakeOrderedAndProject (58) -+- * Project (57) - +- Window (56) - +- * Sort (55) - +- Exchange (54) - +- * HashAggregate (53) - +- Exchange (52) - +- * HashAggregate (51) - +- Union (50) - :- * HashAggregate (39) - : +- Exchange (38) - : +- * HashAggregate (37) - : +- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- BroadcastExchange (34) - : +- * BroadcastHashJoin LeftSemi BuildRight (33) - : :- * ColumnarToRow (12) - : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (10) - : +- BroadcastExchange (32) - : +- * Project (31) - : +- * Filter (30) - : +- Window (29) - : +- WindowGroupLimit (28) - : +- * ColumnarToRow (27) - : +- CometSort (26) - : +- CometHashAggregate (25) - : +- CometExchange (24) - : +- CometHashAggregate (23) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - : +- ReusedExchange (20) - :- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * HashAggregate (41) - : +- ReusedExchange (40) - +- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- * HashAggregate (46) - +- ReusedExchange (45) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +TakeOrderedAndProject (66) ++- * Project (65) + +- Window (64) + +- * CometColumnarToRow (63) + +- CometSort (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometColumnarExchange (59) + +- * HashAggregate (58) + +- Union (57) + :- * HashAggregate (42) + : +- * CometColumnarToRow (41) + : +- CometColumnarExchange (40) + : +- * HashAggregate (39) + : +- * Project (38) + : +- * BroadcastHashJoin Inner BuildRight (37) + : :- * CometColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (36) + : +- * Project (35) + : +- * BroadcastHashJoin LeftSemi BuildRight (34) + : :- * CometColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometNativeScan parquet spark_catalog.default.store (10) + : +- BroadcastExchange (33) + : +- * Project (32) + : +- * Filter (31) + : +- Window (30) + : +- WindowGroupLimit (29) + : +- * CometColumnarToRow (28) + : +- CometSort (27) + : +- CometHashAggregate (26) + : +- CometExchange (25) + : +- CometHashAggregate (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.store (15) + : +- ReusedExchange (21) + :- * HashAggregate (49) + : +- * CometColumnarToRow (48) + : +- CometColumnarExchange (47) + : +- * HashAggregate (46) + : +- * HashAggregate (45) + : +- * CometColumnarToRow (44) + : +- ReusedExchange (43) + +- * HashAggregate (56) + +- * CometColumnarToRow (55) + +- CometColumnarExchange (54) + +- * HashAggregate (53) + +- * HashAggregate (52) + +- * CometColumnarToRow (51) + +- ReusedExchange (50) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -92,233 +107,267 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) ColumnarToRow [codegen id : 4] +(9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometNativeScan: `spark_catalog`.`default`.`store` +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: [s_store_sk#6, s_county#7, s_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (11) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] Condition : isnotnull(s_store_sk#6) -(12) ColumnarToRow [codegen id : 3] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Arguments: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (14) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(15) CometNativeScan: `spark_catalog`.`default`.`store` +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#12, s_state#13] Condition : isnotnull(s_store_sk#12) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] -(18) CometBroadcastHashJoin +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] +Right output [2]: [s_store_sk#12, s_state#14] Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] -(23) CometHashAggregate -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -(24) CometExchange -Input [2]: [s_state#13, sum#15] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) CometHashAggregate -Input [2]: [s_state#13, sum#15] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -(26) CometSort -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13, _w0#16, s_state#13], [s_state#13 ASC NULLS FIRST, _w0#16 DESC NULLS LAST] +(27) CometSort +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(27) ColumnarToRow [codegen id : 1] -Input [3]: [s_state#13, _w0#16, s_state#13] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#14, _w0#17, s_state#14] -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13], [_w0#16 DESC NULLS LAST], rank(_w0#16), 5, Final +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14], [_w0#17 DESC NULLS LAST], rank(_w0#17), 5, Final -(29) Window -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [rank(_w0#16) windowspecdefinition(s_state#13, _w0#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#17], [s_state#13], [_w0#16 DESC NULLS LAST] +(30) Window +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] -(30) Filter [codegen id : 2] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] -Condition : (ranking#17 <= 5) +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) -(31) Project [codegen id : 2] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] -(32) BroadcastExchange -Input [1]: [s_state#13] +(33) BroadcastExchange +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] -(33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None -(34) BroadcastExchange +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 4] +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(36) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_county#7, s_state#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_county#7, s_state#19] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] -(37) HashAggregate [codegen id : 4] -Input [3]: [ss_net_profit#2, s_county#7, s_state#8] -Keys [2]: [s_state#8, s_county#7] +(39) HashAggregate [codegen id : 4] +Input [3]: [ss_net_profit#2, s_county#7, s_state#19] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [s_state#8, s_county#7, sum#19] +Aggregate Attributes [1]: [sum#20] +Results [3]: [s_state#19, s_county#7, sum#21] + +(40) CometColumnarExchange +Input [3]: [s_state#19, s_county#7, sum#21] +Arguments: hashpartitioning(s_state#19, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) Exchange -Input [3]: [s_state#8, s_county#7, sum#19] -Arguments: hashpartitioning(s_state#8, s_county#7, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(41) CometColumnarToRow [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] -(39) HashAggregate [codegen id : 5] -Input [3]: [s_state#8, s_county#7, sum#19] -Keys [2]: [s_state#8, s_county#7] +(42) HashAggregate [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#20] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#20,17,2) as decimal(27,2)) AS total_sum#21, s_state#8, s_county#7, 0 AS g_state#22, 0 AS g_county#23, 0 AS lochierarchy#24] - -(40) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_state#25, s_county#26, sum#27] - -(41) HashAggregate [codegen id : 10] -Input [3]: [s_state#25, s_county#26, sum#27] -Keys [2]: [s_state#25, s_county#26] -Functions [1]: [sum(UnscaledValue(ss_net_profit#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#28))#20] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#28))#20,17,2) AS total_sum#29, s_state#25] - -(42) HashAggregate [codegen id : 10] -Input [2]: [total_sum#29, s_state#25] -Keys [1]: [s_state#25] -Functions [1]: [partial_sum(total_sum#29)] -Aggregate Attributes [2]: [sum#30, isEmpty#31] -Results [3]: [s_state#25, sum#32, isEmpty#33] - -(43) Exchange -Input [3]: [s_state#25, sum#32, isEmpty#33] -Arguments: hashpartitioning(s_state#25, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(44) HashAggregate [codegen id : 11] -Input [3]: [s_state#25, sum#32, isEmpty#33] -Keys [1]: [s_state#25] -Functions [1]: [sum(total_sum#29)] -Aggregate Attributes [1]: [sum(total_sum#29)#34] -Results [6]: [sum(total_sum#29)#34 AS total_sum#35, s_state#25, null AS s_county#36, 0 AS g_state#37, 1 AS g_county#38, 1 AS lochierarchy#39] - -(45) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_state#40, s_county#41, sum#42] - -(46) HashAggregate [codegen id : 16] -Input [3]: [s_state#40, s_county#41, sum#42] -Keys [2]: [s_state#40, s_county#41] -Functions [1]: [sum(UnscaledValue(ss_net_profit#43))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#43))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#43))#20,17,2) AS total_sum#44] - -(47) HashAggregate [codegen id : 16] -Input [1]: [total_sum#44] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#19 AS s_state#24, s_county#7 AS s_county#25, 0 AS g_state#26, 0 AS g_county#27, 0 AS lochierarchy#28] + +(43) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#19, s_county#29, sum#30] + +(44) CometColumnarToRow [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] + +(45) HashAggregate [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] +Keys [2]: [s_state#19, s_county#29] +Functions [1]: [sum(UnscaledValue(ss_net_profit#31))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#31))#22] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#31))#22,17,2) AS total_sum#32, s_state#19] + +(46) HashAggregate [codegen id : 10] +Input [2]: [total_sum#32, s_state#19] +Keys [1]: [s_state#19] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#33, isEmpty#34] +Results [3]: [s_state#19, sum#35, isEmpty#36] + +(47) CometColumnarExchange +Input [3]: [s_state#19, sum#35, isEmpty#36] +Arguments: hashpartitioning(s_state#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(48) CometColumnarToRow [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] + +(49) HashAggregate [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Keys [1]: [s_state#19] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#37] +Results [6]: [sum(total_sum#32)#37 AS total_sum#38, s_state#19, null AS s_county#39, 0 AS g_state#40, 1 AS g_county#41, 1 AS lochierarchy#42] + +(50) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#19, s_county#43, sum#44] + +(51) CometColumnarToRow [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] + +(52) HashAggregate [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] +Keys [2]: [s_state#19, s_county#43] +Functions [1]: [sum(UnscaledValue(ss_net_profit#45))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#45))#22] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#45))#22,17,2) AS total_sum#46] + +(53) HashAggregate [codegen id : 16] +Input [1]: [total_sum#46] Keys: [] -Functions [1]: [partial_sum(total_sum#44)] -Aggregate Attributes [2]: [sum#45, isEmpty#46] -Results [2]: [sum#47, isEmpty#48] +Functions [1]: [partial_sum(total_sum#46)] +Aggregate Attributes [2]: [sum#47, isEmpty#48] +Results [2]: [sum#49, isEmpty#50] -(48) Exchange -Input [2]: [sum#47, isEmpty#48] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(54) CometColumnarExchange +Input [2]: [sum#49, isEmpty#50] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(49) HashAggregate [codegen id : 17] -Input [2]: [sum#47, isEmpty#48] +(55) CometColumnarToRow [codegen id : 17] +Input [2]: [sum#49, isEmpty#50] + +(56) HashAggregate [codegen id : 17] +Input [2]: [sum#49, isEmpty#50] Keys: [] -Functions [1]: [sum(total_sum#44)] -Aggregate Attributes [1]: [sum(total_sum#44)#49] -Results [6]: [sum(total_sum#44)#49 AS total_sum#50, null AS s_state#51, null AS s_county#52, 1 AS g_state#53, 1 AS g_county#54, 2 AS lochierarchy#55] +Functions [1]: [sum(total_sum#46)] +Aggregate Attributes [1]: [sum(total_sum#46)#51] +Results [6]: [sum(total_sum#46)#51 AS total_sum#52, null AS s_state#53, null AS s_county#54, 1 AS g_state#55, 1 AS g_county#56, 2 AS lochierarchy#57] -(50) Union +(57) Union -(51) HashAggregate [codegen id : 18] -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Keys [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] +(58) HashAggregate [codegen id : 18] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] +Results [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] -(52) Exchange -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Arguments: hashpartitioning(total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(59) CometColumnarExchange +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Arguments: hashpartitioning(total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(53) HashAggregate [codegen id : 19] -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Keys [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] +(60) CometHashAggregate +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, CASE WHEN (g_county#23 = 0) THEN s_state#8 END AS _w0#56] -(54) Exchange -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: hashpartitioning(lochierarchy#24, _w0#56, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(61) CometExchange +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: hashpartitioning(lochierarchy#28, _w0#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(62) CometSort +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58], [lochierarchy#28 ASC NULLS FIRST, _w0#58 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] -(55) Sort [codegen id : 20] -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: [lochierarchy#24 ASC NULLS FIRST, _w0#56 ASC NULLS FIRST, total_sum#21 DESC NULLS LAST], false, 0 +(63) CometColumnarToRow [codegen id : 19] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] -(56) Window -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: [rank(total_sum#21) windowspecdefinition(lochierarchy#24, _w0#56, total_sum#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#57], [lochierarchy#24, _w0#56], [total_sum#21 DESC NULLS LAST] +(64) Window +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#28, _w0#58, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#59], [lochierarchy#28, _w0#58], [total_sum#23 DESC NULLS LAST] -(57) Project [codegen id : 21] -Output [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] -Input [6]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56, rank_within_parent#57] +(65) Project [codegen id : 20] +Output [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] +Input [6]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58, rank_within_parent#59] -(58) TakeOrderedAndProject -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] -Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN s_state#8 END ASC NULLS FIRST, rank_within_parent#57 ASC NULLS FIRST], [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] +(66) TakeOrderedAndProject +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#24 END ASC NULLS FIRST, rank_within_parent#59 ASC NULLS FIRST], [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/extended.txt new file mode 100644 index 0000000000..317d6a05b9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/extended.txt @@ -0,0 +1,155 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Filter + : +- Window + : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : +- CometColumnarToRow + : +- CometSort + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Filter + : +- Window + : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : +- CometColumnarToRow + : +- CometSort + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 93 out of 138 eligible operators (67%). Final plan contains 15 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/simplified.txt index 03fbe0bb70..bcf31a302f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/simplified.txt @@ -1,86 +1,92 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (21) + WholeStageCodegen (20) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (20) - Sort [lochierarchy,_w0,total_sum] + WholeStageCodegen (19) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (19) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - WholeStageCodegen (18) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] + CometSort [total_sum,s_state,s_county,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_state] [total_sum,s_state,s_county,lochierarchy,_w0,g_county] + CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 + WholeStageCodegen (18) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum] + CometColumnarToRow InputAdapter - Exchange [s_state,s_county] #3 + CometColumnarExchange [s_state,s_county] #3 WholeStageCodegen (4) HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] Project [ss_net_profit,s_county,s_state] BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #7 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #8 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (11) - HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #7 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #8 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (11) + HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [s_state] #9 + CometColumnarExchange [s_state] #9 WholeStageCodegen (10) HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange #10 + CometColumnarExchange #10 WholeStageCodegen (16) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/explain.txt index bdc0aa2e1e..ce0e312071 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/explain.txt @@ -1,79 +1,89 @@ == Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometHashAggregate (60) - +- CometExchange (59) - +- CometHashAggregate (58) - +- CometProject (57) - +- CometSortMergeJoin (56) - :- CometSort (50) - : +- CometExchange (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometBroadcastHashJoin (37) - : : : :- CometProject (33) - : : : : +- CometBroadcastHashJoin (32) - : : : : :- CometProject (27) - : : : : : +- CometBroadcastHashJoin (26) - : : : : : :- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) : : : : : : : :- CometProject (12) : : : : : : : : +- CometBroadcastHashJoin (11) : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (3) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : : : : : : : +- ReusedExchange (13) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (16) - : : : : : +- CometBroadcastExchange (25) - : : : : : +- CometProject (24) - : : : : : +- CometFilter (23) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (22) - : : : : +- CometBroadcastExchange (31) - : : : : +- CometProject (30) - : : : : +- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (28) - : : : +- CometBroadcastExchange (36) - : : : +- CometFilter (35) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (44) - +- CometSort (55) - +- CometExchange (54) - +- CometProject (53) - +- CometFilter (52) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (51) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct (2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(3) CometNativeScan: `spark_catalog`.`default`.`inventory` +(3) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#12)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct (4) CometFilter Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] @@ -92,9 +102,12 @@ Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct (9) CometFilter Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] @@ -113,211 +126,244 @@ Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -(13) ReusedExchange [Reuses operator id: 10] +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_desc#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) -(14) CometBroadcastHashJoin +(15) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] + +(16) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] Right output [2]: [i_item_sk#15, i_item_desc#16] Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight -(15) CometProject +(17) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(16) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17, cd_marital_status#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct -(17) CometFilter +(19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = M)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#17)) -(18) CometProject +(20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] Arguments: [cd_demo_sk#17], [cd_demo_sk#17] -(19) CometBroadcastExchange +(21) CometBroadcastExchange Input [1]: [cd_demo_sk#17] Arguments: [cd_demo_sk#17] -(20) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [1]: [cd_demo_sk#17] Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight -(21) CometProject +(23) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(22) CometNativeScan: `spark_catalog`.`default`.`household_demographics` +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19, hd_buy_potential#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk)] +ReadSchema: struct -(23) CometFilter +(25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#19)) -(24) CometProject +(26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] Arguments: [hd_demo_sk#19], [hd_demo_sk#19] -(25) CometBroadcastExchange +(27) CometBroadcastExchange Input [1]: [hd_demo_sk#19] Arguments: [hd_demo_sk#19] -(26) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [1]: [hd_demo_sk#19] Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight -(27) CometProject +(29) CometProject Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(28) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] +ReadSchema: struct -(29) CometFilter +(31) CometFilter Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) -(30) CometProject +(32) CometProject Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] -(31) CometBroadcastExchange +(33) CometBroadcastExchange Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] -(32) CometBroadcastHashJoin +(34) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight -(33) CometProject +(35) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct -(35) CometFilter +(37) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(36) CometBroadcastExchange +(38) CometBroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] Arguments: [d_date_sk#25, d_week_seq#26] -(37) CometBroadcastHashJoin +(39) CometBroadcastHashJoin Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] Right output [2]: [d_date_sk#25, d_week_seq#26] Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight -(38) CometProject +(40) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] +ReadSchema: struct -(40) CometFilter +(42) CometFilter Input [2]: [d_date_sk#27, d_date#28] Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(41) CometBroadcastExchange +(43) CometBroadcastExchange Input [2]: [d_date_sk#27, d_date#28] Arguments: [d_date_sk#27, d_date#28] -(42) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] Right output [2]: [d_date_sk#27, d_date#28] Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight -(43) CometProject +(45) CometProject Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(44) CometNativeScan: `spark_catalog`.`default`.`promotion` +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct -(45) CometFilter +(47) CometFilter Input [1]: [p_promo_sk#29] Condition : isnotnull(p_promo_sk#29) -(46) CometBroadcastExchange +(48) CometBroadcastExchange Input [1]: [p_promo_sk#29] Arguments: [p_promo_sk#29] -(47) CometBroadcastHashJoin +(49) CometBroadcastHashJoin Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Right output [1]: [p_promo_sk#29] Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight -(48) CometProject +(50) CometProject Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(49) CometExchange +(51) CometExchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(50) CometSort +(52) CometSort Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(51) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct -(52) CometFilter +(54) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(53) CometProject +(55) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] -(54) CometExchange +(56) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(55) CometSort +(57) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] -(56) CometSortMergeJoin +(58) CometSortMergeJoin Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Right output [2]: [cr_item_sk#30, cr_order_number#31] Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter -(57) CometProject +(59) CometProject Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(58) CometHashAggregate +(60) CometHashAggregate Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] -(59) CometExchange +(61) CometExchange Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(60) CometHashAggregate +(62) CometHashAggregate Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] -(61) CometTakeOrderedAndProject +(63) CometTakeOrderedAndProject Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -(62) ColumnarToRow [codegen id : 1] +(64) CometColumnarToRow [codegen id : 1] Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/extended.txt new file mode 100644 index 0000000000..3bf43ef8c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/extended.txt @@ -0,0 +1,66 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_returns + +Comet accelerated 63 out of 63 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/simplified.txt index e70defef63..073bd57a8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] CometProject [w_warehouse_name,i_item_desc,d_week_seq] @@ -28,37 +28,39 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [i_item_sk,i_item_desc] #4 - CometBroadcastExchange [cd_demo_sk] #5 + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #6 + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #7 + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #8 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #9 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #10 + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #11 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/explain.txt index 2b7ef29c29..d06220c1f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/explain.txt @@ -1,232 +1,383 @@ == Physical Plan == -* ColumnarToRow (44) -+- CometTakeOrderedAndProject (43) - +- CometProject (42) - +- CometBroadcastHashJoin (41) - :- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - +- CometBroadcastExchange (40) - +- CometHashAggregate (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(13) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] - -(14) CometExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] - -(16) CometFilter -Input [2]: [customer_id#11, year_total#12] -Condition : (isnotnull(year_total#12) AND (year_total#12 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Arguments: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] - -(18) CometFilter -Input [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Condition : (isnotnull(c_customer_sk#13) AND isnotnull(c_customer_id#14)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] - -(20) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Right output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_sk#13], [ss_customer_sk#17], Inner, BuildRight - -(21) CometProject -Input [7]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16, ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(23) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : (((isnotnull(d_year#21) AND (d_year#21 = 2002)) AND d_year#21 IN (2001,2002)) AND isnotnull(d_date_sk#20)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(25) CometBroadcastHashJoin -Left output [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight - -(26) CometProject -Input [7]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] - -(27) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#18))] - -(28) CometExchange -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Arguments: hashpartitioning(c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) + +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) + +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] + +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight + +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] + +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) + +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] + +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] + +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight + +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] + +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] + +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight + +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] (29) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [sum(UnscaledValue(ss_net_paid#18))] - -(30) CometBroadcastExchange -Input [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#11, year_total#12] -Right output [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#11], [customer_id#23], Inner, BuildRight - -(32) ReusedExchange [Reuses operator id: 14] -Output [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] - -(33) CometHashAggregate -Input [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] -Keys [4]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30] -Functions [1]: [sum(UnscaledValue(ws_net_paid#32))] - -(34) CometFilter -Input [2]: [customer_id#33, year_total#34] -Condition : (isnotnull(year_total#34) AND (year_total#34 > 0.00)) - -(35) CometBroadcastExchange -Input [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#33, year_total#34] - -(36) CometBroadcastHashJoin -Left output [6]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Right output [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#11], [customer_id#33], Inner, BuildRight - -(37) CometProject -Input [8]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, customer_id#33, year_total#34] -Arguments: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34], [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] - -(38) ReusedExchange [Reuses operator id: 28] -Output [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] - -(39) CometHashAggregate -Input [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] -Keys [4]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(40) CometBroadcastExchange -Input [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#41, year_total#42] - -(41) CometBroadcastHashJoin -Left output [7]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] -Right output [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#11], [customer_id#41], Inner, (CASE WHEN (year_total#34 > 0.00) THEN (year_total#42 / year_total#34) END > CASE WHEN (year_total#12 > 0.00) THEN (year_total#26 / year_total#12) END), BuildRight - -(42) CometProject -Input [9]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34, customer_id#41, year_total#42] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(43) CometTakeOrderedAndProject -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#24 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST,customer_last_name#25 ASC NULLS FIRST], output=[customer_id#23,customer_first_name#24,customer_last_name#25]), [customer_id#23, customer_first_name#24, customer_last_name#25], 100, [customer_first_name#24 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST, customer_last_name#25 ASC NULLS FIRST], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(44) ColumnarToRow [codegen id : 1] -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight + +(34) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) + +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] + +(37) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) + +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] + +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight + +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] + +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] + +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight + +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] + +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] + +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] + +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] + +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight + +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#27 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_last_name#28 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/extended.txt new file mode 100644 index 0000000000..bcd6c3e73d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 75 out of 75 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/simplified.txt index ea904874f9..95b5b340e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometProject [customer_id,customer_first_name,customer_last_name] @@ -8,39 +8,63 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #1 - CometBroadcastExchange [customer_id,year_total] #8 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #5 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/explain.txt index 1c6fedd6e0..7239df4e84 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/explain.txt @@ -1,18 +1,18 @@ == Physical Plan == -* ColumnarToRow (86) -+- CometTakeOrderedAndProject (85) - +- CometProject (84) - +- CometSortMergeJoin (83) - :- CometSort (44) - : +- CometExchange (43) - : +- CometFilter (42) - : +- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometUnion (35) +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometProject (122) + +- CometSortMergeJoin (121) + :- CometSort (66) + : +- CometExchange (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) : :- CometProject (22) : : +- CometSortMergeJoin (21) : : :- CometSort (15) @@ -22,86 +22,131 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : :- CometProject (28) - : : +- CometSortMergeJoin (27) - : : :- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometSort (26) - : : +- ReusedExchange (25) - : +- CometProject (34) - : +- CometSortMergeJoin (33) - : :- CometSort (30) - : : +- ReusedExchange (29) - : +- CometSort (32) - : +- ReusedExchange (31) - +- CometSort (82) - +- CometExchange (81) - +- CometFilter (80) - +- CometHashAggregate (79) - +- CometExchange (78) - +- CometHashAggregate (77) - +- CometHashAggregate (76) - +- CometExchange (75) - +- CometHashAggregate (74) - +- CometUnion (73) - :- CometProject (60) - : +- CometSortMergeJoin (59) - : :- CometSort (56) - : : +- CometExchange (55) - : : +- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometProject (49) - : : : +- CometBroadcastHashJoin (48) - : : : :- CometFilter (46) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (45) - : : : +- ReusedExchange (47) - : : +- CometBroadcastExchange (52) - : : +- CometFilter (51) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (50) - : +- CometSort (58) - : +- ReusedExchange (57) - :- CometProject (66) - : +- CometSortMergeJoin (65) - : :- CometSort (62) - : : +- ReusedExchange (61) - : +- CometSort (64) - : +- ReusedExchange (63) - +- CometProject (72) - +- CometSortMergeJoin (71) - :- CometSort (68) - : +- ReusedExchange (67) - +- CometSort (70) - +- ReusedExchange (69) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) + +- CometSort (120) + +- CometExchange (119) + +- CometFilter (118) + +- CometHashAggregate (117) + +- CometExchange (116) + +- CometHashAggregate (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometProject (82) + : +- CometSortMergeJoin (81) + : :- CometSort (78) + : : +- CometExchange (77) + : : +- CometProject (76) + : : +- CometBroadcastHashJoin (75) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometFilter (68) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (67) + : : : +- ReusedExchange (69) + : : +- CometBroadcastExchange (74) + : : +- CometFilter (73) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) + : +- CometSort (80) + : +- ReusedExchange (79) + :- CometProject (96) + : +- CometSortMergeJoin (95) + : :- CometSort (92) + : : +- CometExchange (91) + : : +- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometFilter (84) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (83) + : : : +- ReusedExchange (85) + : : +- ReusedExchange (88) + : +- CometSort (94) + : +- ReusedExchange (93) + +- CometProject (110) + +- CometSortMergeJoin (109) + :- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometBroadcastHashJoin (103) + : :- CometProject (101) + : : +- CometBroadcastHashJoin (100) + : : :- CometFilter (98) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (97) + : : +- ReusedExchange (99) + : +- ReusedExchange (102) + +- CometSort (108) + +- ReusedExchange (107) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct (2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] @@ -120,9 +165,12 @@ Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct (10) CometFilter Input [2]: [d_date_sk#12, d_year#13] @@ -149,9 +197,12 @@ Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREM Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct (17) CometFilter Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] @@ -178,266 +229,453 @@ Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14 Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(23) ReusedExchange [Reuses operator id: 14] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] +(23) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct -(24) CometSort -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] +(24) CometFilter +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) -(25) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] +(25) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(26) CometSort -Input [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33], [sr_ticket_number#31 ASC NULLS FIRST, sr_item_sk#30 ASC NULLS FIRST] +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight -(27) CometSortMergeJoin -Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Right output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#31, sr_item_sk#30], LeftOuter +(27) CometProject +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(28) CometProject -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29, sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35], [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, (ss_quantity#23 - coalesce(sr_return_quantity#32, 0)) AS sales_cnt#34, (ss_ext_sales_price#24 - coalesce(sr_return_amt#33, 0.00)) AS sales_amt#35] +(28) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#31, d_year#32] -(29) ReusedExchange [Reuses operator id: 14] -Output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight -(30) CometSort -Input [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Arguments: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44], [ws_order_number#37 ASC NULLS FIRST, ws_item_sk#36 ASC NULLS FIRST] +(30) CometProject +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -(31) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] +(31) CometExchange +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48], [wr_order_number#46 ASC NULLS FIRST, wr_item_sk#45 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] + +(33) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) + +(35) CometProject +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] + +(36) CometExchange +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(37) CometSort +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] + +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter + +(39) CometProject +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] + +(40) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(41) CometFilter +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) + +(42) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight + +(44) CometProject +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#50, d_year#51] + +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight + +(47) CometProject +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] + +(48) CometExchange +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(49) CometSort +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] + +(50) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) + +(52) CometProject +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] + +(53) CometExchange +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(33) CometSortMergeJoin -Left output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Right output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [ws_order_number#37, ws_item_sk#36], [wr_order_number#46, wr_item_sk#45], LeftOuter +(54) CometSort +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] -(34) CometProject -Input [13]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44, wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50], [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, (ws_quantity#38 - coalesce(wr_return_quantity#47, 0)) AS sales_cnt#49, (ws_ext_sales_price#39 - coalesce(wr_return_amt#48, 0.00)) AS sales_amt#50] +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter -(35) CometUnion +(56) CometProject +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] + +(57) CometUnion Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] -(36) CometHashAggregate +(58) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -(37) CometExchange +(59) CometExchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(38) CometHashAggregate +(60) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -(39) CometHashAggregate +(61) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -(40) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(41) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] +(63) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -(42) CometFilter -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Condition : isnotnull(sales_cnt#53) - -(43) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(44) CometSort -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] - -(45) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] - -(46) CometFilter -Input [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Condition : isnotnull(cs_item_sk#55) - -(47) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(48) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Right output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55], [i_item_sk#60], Inner, BuildRight - -(49) CometProject -Input [10]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(50) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(51) CometFilter -Input [2]: [d_date_sk#65, d_year#66] -Condition : ((isnotnull(d_year#66) AND (d_year#66 = 2001)) AND isnotnull(d_date_sk#65)) - -(52) CometBroadcastExchange -Input [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(53) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [cs_sold_date_sk#59], [d_date_sk#65], Inner, BuildRight - -(54) CometProject -Input [11]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_date_sk#65, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] - -(55) CometExchange -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(56) CometSort -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_order_number#56 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST] - -(57) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] - -(58) CometSort -Input [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70], [cr_order_number#68 ASC NULLS FIRST, cr_item_sk#67 ASC NULLS FIRST] - -(59) CometSortMergeJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Right output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cs_order_number#56, cs_item_sk#55], [cr_order_number#68, cr_item_sk#67], LeftOuter - -(60) CometProject -Input [13]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66, cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20], [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, (cs_quantity#57 - coalesce(cr_return_quantity#69, 0)) AS sales_cnt#19, (cs_ext_sales_price#58 - coalesce(cr_return_amount#70, 0.00)) AS sales_amt#20] - -(61) ReusedExchange [Reuses operator id: 55] -Output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] - -(62) CometSort -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79], [ss_ticket_number#72 ASC NULLS FIRST, ss_item_sk#71 ASC NULLS FIRST] - -(63) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] - -(64) CometSort -Input [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83], [sr_ticket_number#81 ASC NULLS FIRST, sr_item_sk#80 ASC NULLS FIRST] - -(65) CometSortMergeJoin -Left output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Right output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [ss_ticket_number#72, ss_item_sk#71], [sr_ticket_number#81, sr_item_sk#80], LeftOuter - -(66) CometProject -Input [13]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79, sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35], [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, (ss_quantity#73 - coalesce(sr_return_quantity#82, 0)) AS sales_cnt#34, (ss_ext_sales_price#74 - coalesce(sr_return_amt#83, 0.00)) AS sales_amt#35] - -(67) ReusedExchange [Reuses operator id: 55] -Output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] - -(68) CometSort -Input [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Arguments: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92], [ws_order_number#85 ASC NULLS FIRST, ws_item_sk#84 ASC NULLS FIRST] - -(69) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] - -(70) CometSort -Input [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96], [wr_order_number#94 ASC NULLS FIRST, wr_item_sk#93 ASC NULLS FIRST] - -(71) CometSortMergeJoin -Left output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Right output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [ws_order_number#85, ws_item_sk#84], [wr_order_number#94, wr_item_sk#93], LeftOuter - -(72) CometProject -Input [13]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92, wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50], [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, (ws_quantity#86 - coalesce(wr_return_quantity#95, 0)) AS sales_cnt#49, (ws_ext_sales_price#87 - coalesce(wr_return_amt#96, 0.00)) AS sales_amt#50] - -(73) CometUnion -Child 0 Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50] - -(74) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] +(64) CometFilter +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) + +(65) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(66) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] + +(67) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(68) CometFilter +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) + +(69) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] + +(70) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight + +(71) CometProject +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] + +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#73, d_year#74] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(73) CometFilter +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) + +(74) CometBroadcastExchange +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] + +(75) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight + +(76) CometProject +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] + +(77) CometExchange +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(78) CometSort +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] + +(79) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] + +(80) CometSort +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] + +(81) CometSortMergeJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter + +(82) CometProject +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] + +(83) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(84) CometFilter +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) + +(85) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] + +(86) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight + +(87) CometProject +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] + +(88) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#89, d_year#90] + +(89) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight + +(90) CometProject +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] + +(91) CometExchange +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(92) CometSort +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] + +(93) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] + +(94) CometSort +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] + +(95) CometSortMergeJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter + +(96) CometProject +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] + +(97) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(98) CometFilter +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) + +(99) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] + +(100) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight + +(101) CometProject +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] + +(102) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#105, d_year#106] + +(103) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight + +(104) CometProject +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] + +(105) CometExchange +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(106) CometSort +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] + +(107) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] + +(108) CometSort +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] + +(109) CometSortMergeJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter + +(110) CometProject +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] + +(111) CometUnion +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] + +(112) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -(75) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(113) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(76) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] +(114) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -(77) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] +(115) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -(78) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(116) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(79) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] +(117) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -(80) CometFilter -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Condition : isnotnull(sales_cnt#98) +(118) CometFilter +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) -(81) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +(119) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(82) CometSort -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99], [i_brand_id#61 ASC NULLS FIRST, i_class_id#62 ASC NULLS FIRST, i_category_id#63 ASC NULLS FIRST, i_manufact_id#64 ASC NULLS FIRST] +(120) CometSort +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] -(83) CometSortMergeJoin -Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Right output [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], Inner, ((cast(sales_cnt#53 as decimal(17,2)) / cast(sales_cnt#98 as decimal(17,2))) < 0.90000000000000000000) +(121) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) -(84) CometProject -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54, d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], [d_year#66 AS prev_year#100, d_year#13 AS year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#98 AS prev_yr_cnt#102, sales_cnt#53 AS curr_yr_cnt#103, (sales_cnt#53 - sales_cnt#98) AS sales_cnt_diff#104, (sales_amt#54 - sales_amt#99) AS sales_amt_diff#105] +(122) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] -(85) CometTakeOrderedAndProject -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#104 ASC NULLS FIRST,sales_amt_diff#105 ASC NULLS FIRST], output=[prev_year#100,year#101,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#102,curr_yr_cnt#103,sales_cnt_diff#104,sales_amt_diff#105]), [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], 100, [sales_cnt_diff#104 ASC NULLS FIRST, sales_amt_diff#105 ASC NULLS FIRST], [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] +(123) CometTakeOrderedAndProject +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST,sales_amt_diff#119 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST, sales_amt_diff#119 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] -(86) ColumnarToRow [codegen id : 1] -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] +(124) CometColumnarToRow [codegen id : 1] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/extended.txt new file mode 100644 index 0000000000..84620b41ae --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/extended.txt @@ -0,0 +1,158 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_returns + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_returns + +- CometSort + +- CometExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_returns + +Comet accelerated 155 out of 155 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/simplified.txt index 6503d71685..588e8943c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] @@ -7,9 +7,9 @@ WholeStageCodegen (1) CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -23,66 +23,104 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #8 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #9 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #10 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #11 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometBroadcastExchange [d_date_sk,d_year] #12 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/explain.txt index cfbc95b886..8c51c6159e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/explain.txt @@ -1,83 +1,127 @@ == Physical Plan == -TakeOrderedAndProject (66) -+- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- Union (62) - :- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- Union (48) - : :- * ColumnarToRow (22) - : : +- CometProject (21) - : : +- CometBroadcastHashJoin (20) - : : :- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : : +- CometBroadcastExchange (19) - : : +- CometHashAggregate (18) - : : +- ReusedExchange (17) - : :- * Project (41) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (40) - : : :- BroadcastExchange (31) - : : : +- * ColumnarToRow (30) - : : : +- CometHashAggregate (29) - : : : +- CometExchange (28) - : : : +- CometHashAggregate (27) - : : : +- CometProject (26) - : : : +- CometBroadcastHashJoin (25) - : : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (23) - : : : +- ReusedExchange (24) - : : +- * ColumnarToRow (39) - : : +- CometHashAggregate (38) - : : +- CometExchange (37) - : : +- CometHashAggregate (36) - : : +- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (32) - : : +- ReusedExchange (33) - : +- * ColumnarToRow (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometHashAggregate (43) - : : +- ReusedExchange (42) - : +- ReusedExchange (44) - :- * HashAggregate (56) - : +- Exchange (55) - : +- * HashAggregate (54) - : +- * HashAggregate (53) - : +- ReusedExchange (52) - +- * HashAggregate (61) - +- Exchange (60) - +- * HashAggregate (59) - +- * HashAggregate (58) - +- ReusedExchange (57) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (103) ++- CometTakeOrderedAndProject (102) + +- CometHashAggregate (101) + +- CometColumnarExchange (100) + +- * HashAggregate (99) + +- Union (98) + :- * HashAggregate (83) + : +- * CometColumnarToRow (82) + : +- CometColumnarExchange (81) + : +- * HashAggregate (80) + : +- Union (79) + : :- * CometColumnarToRow (31) + : : +- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.store (9) + : : +- CometBroadcastExchange (28) + : : +- CometHashAggregate (27) + : : +- CometExchange (26) + : : +- CometHashAggregate (25) + : : +- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (21) + : : : +- CometBroadcastHashJoin (20) + : : : :- CometFilter (18) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (17) + : : : +- ReusedExchange (19) + : : +- ReusedExchange (22) + : :- * Project (50) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : : :- BroadcastExchange (40) + : : : +- * CometColumnarToRow (39) + : : : +- CometHashAggregate (38) + : : : +- CometExchange (37) + : : : +- CometHashAggregate (36) + : : : +- CometProject (35) + : : : +- CometBroadcastHashJoin (34) + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (32) + : : : +- ReusedExchange (33) + : : +- * CometColumnarToRow (48) + : : +- CometHashAggregate (47) + : : +- CometExchange (46) + : : +- CometHashAggregate (45) + : : +- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns (41) + : : +- ReusedExchange (42) + : +- * CometColumnarToRow (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometFilter (52) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (51) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometNativeScan parquet spark_catalog.default.web_page (56) + : +- CometBroadcastExchange (75) + : +- CometHashAggregate (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + :- * HashAggregate (90) + : +- * CometColumnarToRow (89) + : +- CometColumnarExchange (88) + : +- * HashAggregate (87) + : +- * HashAggregate (86) + : +- * CometColumnarToRow (85) + : +- ReusedExchange (84) + +- * HashAggregate (97) + +- * CometColumnarToRow (96) + +- CometColumnarExchange (95) + +- * HashAggregate (94) + +- * HashAggregate (93) + +- * CometColumnarToRow (92) + +- ReusedExchange (91) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct (2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#5, d_date#6] @@ -100,9 +144,12 @@ Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -(9) CometNativeScan: `spark_catalog`.`default`.`store` +(9) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (10) CometFilter Input [1]: [s_store_sk#7] @@ -135,228 +182,397 @@ Input [3]: [s_store_sk#7, sum#8, sum#9] Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [s_store_sk#10, sum#11, sum#12] +(17) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct -(18) CometHashAggregate -Input [3]: [s_store_sk#10, sum#11, sum#12] -Keys [1]: [s_store_sk#10] -Functions [2]: [sum(UnscaledValue(sr_return_amt#13)), sum(UnscaledValue(sr_net_loss#14))] +(18) CometFilter +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) -(19) CometBroadcastExchange -Input [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#10, returns#15, profit_loss#16] +(19) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#14] (20) CometBroadcastHashJoin -Left output [3]: [s_store_sk#7, sales#17, profit#18] -Right output [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#7], [s_store_sk#10], LeftOuter, BuildRight +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight (21) CometProject -Input [6]: [s_store_sk#7, sales#17, profit#18, s_store_sk#10, returns#15, profit_loss#16] -Arguments: [channel#19, id#20, sales#17, returns#21, profit#22], [store channel AS channel#19, s_store_sk#7 AS id#20, sales#17, coalesce(returns#15, 0.00) AS returns#21, (profit#18 - coalesce(profit_loss#16, 0.00)) AS profit#22] +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] -(22) ColumnarToRow [codegen id : 1] -Input [5]: [channel#19, id#20, sales#17, returns#21, profit#22] +(22) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#15] -(23) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] +(23) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight -(24) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#27] +(24) CometProject +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] -(25) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [cs_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight +(25) CometHashAggregate +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] -(26) CometProject -Input [5]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26, d_date_sk#27] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25], [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] +(26) CometExchange +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate -Input [3]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#24)), partial_sum(UnscaledValue(cs_net_profit#25))] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] -(28) CometExchange -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Arguments: hashpartitioning(cs_call_center_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometBroadcastExchange +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] -(29) CometHashAggregate -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#24)), sum(UnscaledValue(cs_net_profit#25))] +(29) CometBroadcastHashJoin +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight -(30) ColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] +(30) CometProject +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [channel#22, id#23, sales#20, returns#24, profit#25], [store channel AS channel#22, s_store_sk#7 AS id#23, sales#20, coalesce(returns#18, 0.00) AS returns#24, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#25] -(31) BroadcastExchange -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] -Arguments: IdentityBroadcastMode, [plan_id=3] +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] -(32) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Arguments: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] +(32) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] +ReadSchema: struct (33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#35] +Output [1]: [d_date_sk#30] (34) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Right output [1]: [d_date_sk#35] -Arguments: [cr_returned_date_sk#34], [d_date_sk#35], Inner, BuildRight +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (35) CometProject -Input [4]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34, d_date_sk#35] -Arguments: [cr_return_amount#32, cr_net_loss#33], [cr_return_amount#32, cr_net_loss#33] +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] (36) CometHashAggregate -Input [2]: [cr_return_amount#32, cr_net_loss#33] -Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#32)), partial_sum(UnscaledValue(cr_net_loss#33))] +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] (37) CometExchange -Input [2]: [sum#36, sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate -Input [2]: [sum#36, sum#37] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] + +(39) CometColumnarToRow [codegen id : 2] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] + +(40) BroadcastExchange +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] +Arguments: IdentityBroadcastMode, [plan_id=4] + +(41) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] +ReadSchema: struct + +(42) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#38] + +(43) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight + +(44) CometProject +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] + +(45) CometHashAggregate +Input [2]: [cr_return_amount#35, cr_net_loss#36] +Keys: [] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] + +(46) CometExchange +Input [2]: [sum#39, sum#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(47) CometHashAggregate +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#32)), sum(UnscaledValue(cr_net_loss#33))] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] -(39) ColumnarToRow -Input [2]: [returns#38, profit_loss#39] +(48) CometColumnarToRow +Input [2]: [returns#41, profit_loss#42] -(40) BroadcastNestedLoopJoin [codegen id : 3] +(49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None -(41) Project [codegen id : 3] -Output [5]: [catalog channel AS channel#40, cs_call_center_sk#23 AS id#41, sales#30, returns#38, (profit#31 - profit_loss#39) AS profit#42] -Input [5]: [cs_call_center_sk#23, sales#30, profit#31, returns#38, profit_loss#39] - -(42) ReusedExchange [Reuses operator id: 15] -Output [3]: [wp_web_page_sk#43, sum#44, sum#45] - -(43) CometHashAggregate -Input [3]: [wp_web_page_sk#43, sum#44, sum#45] -Keys [1]: [wp_web_page_sk#43] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#46)), sum(UnscaledValue(ws_net_profit#47))] - -(44) ReusedExchange [Reuses operator id: 19] -Output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] - -(45) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#43, sales#51, profit#52] -Right output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [wp_web_page_sk#43], [wp_web_page_sk#48], LeftOuter, BuildRight - -(46) CometProject -Input [6]: [wp_web_page_sk#43, sales#51, profit#52, wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [channel#53, id#54, sales#51, returns#55, profit#56], [web channel AS channel#53, wp_web_page_sk#43 AS id#54, sales#51, coalesce(returns#49, 0.00) AS returns#55, (profit#52 - coalesce(profit_loss#50, 0.00)) AS profit#56] - -(47) ColumnarToRow [codegen id : 4] -Input [5]: [channel#53, id#54, sales#51, returns#55, profit#56] - -(48) Union - -(49) HashAggregate [codegen id : 5] -Input [5]: [channel#19, id#20, sales#17, returns#21, profit#22] -Keys [2]: [channel#19, id#20] -Functions [3]: [partial_sum(sales#17), partial_sum(returns#21), partial_sum(profit#22)] -Aggregate Attributes [6]: [sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62] -Results [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(50) Exchange -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Arguments: hashpartitioning(channel#19, id#20, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(51) HashAggregate [codegen id : 6] -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [2]: [channel#19, id#20] -Functions [3]: [sum(sales#17), sum(returns#21), sum(profit#22)] -Aggregate Attributes [3]: [sum(sales#17)#69, sum(returns#21)#70, sum(profit#22)#71] -Results [5]: [channel#19, id#20, cast(sum(sales#17)#69 as decimal(37,2)) AS sales#72, cast(sum(returns#21)#70 as decimal(37,2)) AS returns#73, cast(sum(profit#22)#71 as decimal(38,2)) AS profit#74] - -(52) ReusedExchange [Reuses operator id: 50] -Output [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(53) HashAggregate [codegen id : 12] -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [2]: [channel#19, id#20] -Functions [3]: [sum(sales#17), sum(returns#21), sum(profit#22)] -Aggregate Attributes [3]: [sum(sales#17)#69, sum(returns#21)#70, sum(profit#22)#71] -Results [4]: [channel#19, sum(sales#17)#69 AS sales#75, sum(returns#21)#70 AS returns#76, sum(profit#22)#71 AS profit#77] - -(54) HashAggregate [codegen id : 12] -Input [4]: [channel#19, sales#75, returns#76, profit#77] -Keys [1]: [channel#19] -Functions [3]: [partial_sum(sales#75), partial_sum(returns#76), partial_sum(profit#77)] -Aggregate Attributes [6]: [sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83] -Results [7]: [channel#19, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] - -(55) Exchange -Input [7]: [channel#19, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] -Arguments: hashpartitioning(channel#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(56) HashAggregate [codegen id : 13] -Input [7]: [channel#19, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] -Keys [1]: [channel#19] -Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] -Aggregate Attributes [3]: [sum(sales#75)#90, sum(returns#76)#91, sum(profit#77)#92] -Results [5]: [channel#19, null AS id#93, sum(sales#75)#90 AS sales#94, sum(returns#76)#91 AS returns#95, sum(profit#77)#92 AS profit#96] - -(57) ReusedExchange [Reuses operator id: 50] -Output [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(58) HashAggregate [codegen id : 19] -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [2]: [channel#19, id#20] -Functions [3]: [sum(sales#17), sum(returns#21), sum(profit#22)] -Aggregate Attributes [3]: [sum(sales#17)#69, sum(returns#21)#70, sum(profit#22)#71] -Results [3]: [sum(sales#17)#69 AS sales#97, sum(returns#21)#70 AS returns#98, sum(profit#22)#71 AS profit#99] - -(59) HashAggregate [codegen id : 19] -Input [3]: [sales#97, returns#98, profit#99] +(50) Project [codegen id : 3] +Output [5]: [catalog channel AS channel#43, cs_call_center_sk#26 AS id#44, sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] + +(51) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] +PushedFilters: [IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(52) CometFilter +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) + +(53) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#50] + +(54) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight + +(55) CometProject +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] + +(56) CometNativeScan parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(57) CometFilter +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) + +(58) CometBroadcastExchange +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] + +(59) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight + +(60) CometProject +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] + +(61) CometHashAggregate +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] + +(62) CometExchange +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(63) CometHashAggregate +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] + +(64) CometNativeScan parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct + +(65) CometFilter +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) + +(66) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(67) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(68) CometProject +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] + +(69) ReusedExchange [Reuses operator id: 58] +Output [1]: [wp_web_page_sk#59] + +(70) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(71) CometProject +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] + +(72) CometHashAggregate +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] + +(73) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(74) CometHashAggregate +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] + +(75) CometBroadcastExchange +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] + +(76) CometBroadcastHashJoin +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight + +(77) CometProject +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [channel#66, id#67, sales#64, returns#68, profit#69], [web channel AS channel#66, wp_web_page_sk#51 AS id#67, sales#64, coalesce(returns#62, 0.00) AS returns#68, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#69] + +(78) CometColumnarToRow [codegen id : 4] +Input [5]: [channel#66, id#67, sales#64, returns#68, profit#69] + +(79) Union + +(80) HashAggregate [codegen id : 5] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] +Keys [2]: [channel#22, id#23] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#24), partial_sum(profit#25)] +Aggregate Attributes [6]: [sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75] +Results [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(81) CometColumnarExchange +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Arguments: hashpartitioning(channel#22, id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(82) CometColumnarToRow [codegen id : 6] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(83) HashAggregate [codegen id : 6] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [5]: [channel#22, id#23, cast(sum(sales#20)#82 as decimal(37,2)) AS sales#85, cast(sum(returns#24)#83 as decimal(37,2)) AS returns#86, cast(sum(profit#25)#84 as decimal(38,2)) AS profit#87] + +(84) ReusedExchange [Reuses operator id: 81] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(85) CometColumnarToRow [codegen id : 12] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(86) HashAggregate [codegen id : 12] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [4]: [channel#22, sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] + +(87) HashAggregate [codegen id : 12] +Input [4]: [channel#22, sales#88, returns#89, profit#90] +Keys [1]: [channel#22] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Results [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] + +(88) CometColumnarExchange +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Arguments: hashpartitioning(channel#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(89) CometColumnarToRow [codegen id : 13] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] + +(90) HashAggregate [codegen id : 13] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Keys [1]: [channel#22] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#103, sum(returns#89)#104, sum(profit#90)#105] +Results [5]: [channel#22, null AS id#106, sum(sales#88)#103 AS sales#107, sum(returns#89)#104 AS returns#108, sum(profit#90)#105 AS profit#109] + +(91) ReusedExchange [Reuses operator id: 81] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(92) CometColumnarToRow [codegen id : 19] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(93) HashAggregate [codegen id : 19] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [3]: [sum(sales#20)#82 AS sales#110, sum(returns#24)#83 AS returns#111, sum(profit#25)#84 AS profit#112] + +(94) HashAggregate [codegen id : 19] +Input [3]: [sales#110, returns#111, profit#112] Keys: [] -Functions [3]: [partial_sum(sales#97), partial_sum(returns#98), partial_sum(profit#99)] -Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Results [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Functions [3]: [partial_sum(sales#110), partial_sum(returns#111), partial_sum(profit#112)] +Aggregate Attributes [6]: [sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118] +Results [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] + +(95) CometColumnarExchange +Input [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(60) Exchange -Input [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(96) CometColumnarToRow [codegen id : 20] +Input [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] -(61) HashAggregate [codegen id : 20] -Input [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +(97) HashAggregate [codegen id : 20] +Input [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] Keys: [] -Functions [3]: [sum(sales#97), sum(returns#98), sum(profit#99)] -Aggregate Attributes [3]: [sum(sales#97)#112, sum(returns#98)#113, sum(profit#99)#114] -Results [5]: [null AS channel#115, null AS id#116, sum(sales#97)#112 AS sales#117, sum(returns#98)#113 AS returns#118, sum(profit#99)#114 AS profit#119] +Functions [3]: [sum(sales#110), sum(returns#111), sum(profit#112)] +Aggregate Attributes [3]: [sum(sales#110)#125, sum(returns#111)#126, sum(profit#112)#127] +Results [5]: [null AS channel#128, null AS id#129, sum(sales#110)#125 AS sales#130, sum(returns#111)#126 AS returns#131, sum(profit#112)#127 AS profit#132] -(62) Union +(98) Union -(63) HashAggregate [codegen id : 21] -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Keys [5]: [channel#19, id#20, sales#72, returns#73, profit#74] +(99) HashAggregate [codegen id : 21] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#19, id#20, sales#72, returns#73, profit#74] +Results [5]: [channel#22, id#23, sales#85, returns#86, profit#87] -(64) Exchange -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Arguments: hashpartitioning(channel#19, id#20, sales#72, returns#73, profit#74, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(100) CometColumnarExchange +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: hashpartitioning(channel#22, id#23, sales#85, returns#86, profit#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] -(65) HashAggregate [codegen id : 22] -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Keys [5]: [channel#19, id#20, sales#72, returns#73, profit#74] +(101) CometHashAggregate +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -(66) TakeOrderedAndProject -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Arguments: 100, [channel#19 ASC NULLS FIRST, id#20 ASC NULLS FIRST], [channel#19, id#20, sales#72, returns#73, profit#74] +(102) CometTakeOrderedAndProject +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#22 ASC NULLS FIRST,id#23 ASC NULLS FIRST], output=[channel#22,id#23,sales#85,returns#86,profit#87]), [channel#22, id#23, sales#85, returns#86, profit#87], 100, 0, [channel#22 ASC NULLS FIRST, id#23 ASC NULLS FIRST], [channel#22, id#23, sales#85, returns#86, profit#87] + +(103) CometColumnarToRow [codegen id : 22] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/extended.txt new file mode 100644 index 0000000000..13d4b5c286 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/extended.txt @@ -0,0 +1,322 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- Project + : +- BroadcastNestedLoopJoin + : :- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometColumnarToRow + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 278 out of 302 eligible operators (92%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/simplified.txt index 1a5f29f9f3..cc8a78ab5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/simplified.txt @@ -1,91 +1,128 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (22) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (21) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (5) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometExchange [s_store_sk] #3 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - ReusedExchange [s_store_sk,sum,sum] #3 - WholeStageCodegen (3) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #8 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedExchange [d_date_sk] #4 - ColumnarToRow +WholeStageCodegen (22) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (21) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (6) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (5) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [s_store_sk,sales,profit,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #3 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 + CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #7 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (3) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #9 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #10 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #9 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (4) - ColumnarToRow + CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #11 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 + CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #14 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 + WholeStageCodegen (13) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #15 + WholeStageCodegen (12) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (20) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #16 + WholeStageCodegen (19) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow InputAdapter - CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - ReusedExchange [wp_web_page_sk,sum,sum] #3 - ReusedExchange [wp_web_page_sk,returns,profit_loss] #6 - WholeStageCodegen (13) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #10 - WholeStageCodegen (12) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (20) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #11 - WholeStageCodegen (19) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/explain.txt index 03ce3dd98b..e26c10beab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * ColumnarToRow (46) - +- CometSortMergeJoin (45) - :- CometProject (27) - : +- CometSortMergeJoin (26) +TakeOrderedAndProject (67) ++- * Project (66) + +- * CometColumnarToRow (65) + +- CometSortMergeJoin (64) + :- CometProject (43) + : +- CometSortMergeJoin (42) : :- CometSort (21) : : +- CometHashAggregate (20) : : +- CometExchange (19) @@ -17,41 +17,64 @@ TakeOrderedAndProject (48) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) : : +- CometBroadcastExchange (15) : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometSort (25) - : +- CometFilter (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - +- CometSort (44) - +- CometFilter (43) - +- CometHashAggregate (42) - +- CometExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometFilter (35) - : +- CometSortMergeJoin (34) - : :- CometSort (31) - : : +- CometExchange (30) - : : +- CometFilter (29) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : +- CometSort (33) - : +- ReusedExchange (32) - +- ReusedExchange (37) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : +- CometSort (41) + : +- CometFilter (40) + : +- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (33) + : : +- CometFilter (32) + : : +- CometSortMergeJoin (31) + : : :- CometSort (25) + : : : +- CometExchange (24) + : : : +- CometFilter (23) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (22) + : : +- CometSort (30) + : : +- CometExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (26) + : +- ReusedExchange (34) + +- CometSort (63) + +- CometFilter (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometFilter (54) + : +- CometSortMergeJoin (53) + : :- CometSort (47) + : : +- CometExchange (46) + : : +- CometFilter (45) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (44) + : +- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometFilter (49) + : +- CometNativeScan parquet spark_catalog.default.catalog_returns (48) + +- ReusedExchange (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -65,9 +88,12 @@ Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct (6) CometFilter Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] @@ -98,9 +124,12 @@ Condition : isnull(sr_ticket_number#9) Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct (14) CometFilter Input [2]: [d_date_sk#11, d_year#12] @@ -137,114 +166,208 @@ Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(22) ReusedExchange [Reuses operator id: 19] -Output [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] +(22) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct -(23) CometHashAggregate -Input [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] -Keys [3]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +(23) CometFilter +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) -(24) CometFilter -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Condition : (coalesce(ws_qty#31, 0) > 0) +(24) CometExchange +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometSort -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33], [ws_sold_year#29 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#30 ASC NULLS FIRST] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(26) CometSortMergeJoin -Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Right output [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30], Inner +(26) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct -(27) CometProject -Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] +(27) CometFilter +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] +(28) CometProject +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] -(29) CometFilter -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Condition : (isnotnull(cs_item_sk#35) AND isnotnull(cs_bill_customer_sk#34)) +(29) CometExchange +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(30) CometExchange -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_order_number#36, cs_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(30) CometSort +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] -(31) CometSort -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_order_number#36 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST] +(31) CometSortMergeJoin +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter -(32) ReusedExchange [Reuses operator id: 8] -Output [2]: [cr_item_sk#41, cr_order_number#42] +(32) CometFilter +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) -(33) CometSort -Input [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cr_item_sk#41, cr_order_number#42], [cr_order_number#42 ASC NULLS FIRST, cr_item_sk#41 ASC NULLS FIRST] +(33) CometProject +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] -(34) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cs_order_number#36, cs_item_sk#35], [cr_order_number#42, cr_item_sk#41], LeftOuter +(34) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#30, d_year#31] -(35) CometFilter -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Condition : isnull(cr_order_number#42) +(35) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight (36) CometProject -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] - -(37) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#43, d_year#44] - -(38) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [cs_sold_date_sk#40], [d_date_sk#43], Inner, BuildRight - -(39) CometProject -Input [8]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, d_date_sk#43, d_year#44] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] - -(40) CometHashAggregate -Input [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [partial_sum(cs_quantity#37), partial_sum(UnscaledValue(cs_wholesale_cost#38)), partial_sum(UnscaledValue(cs_sales_price#39))] - -(41) CometExchange -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Arguments: hashpartitioning(d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(42) CometHashAggregate -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [sum(cs_quantity#37), sum(UnscaledValue(cs_wholesale_cost#38)), sum(UnscaledValue(cs_sales_price#39))] - -(43) CometFilter -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Condition : (coalesce(cs_qty#50, 0) > 0) - -(44) CometSort -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52], [cs_sold_year#48 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST, cs_customer_sk#49 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] -Right output [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49], Inner - -(46) ColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(47) Project [codegen id : 1] -Output [13]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#31 + cs_qty#50), 1) as double)))), 2) AS ratio#53, ss_qty#17 AS store_qty#54, ss_wc#18 AS store_wholesale_cost#55, ss_sp#19 AS store_sales_price#56, (coalesce(ws_qty#31, 0) + coalesce(cs_qty#50, 0)) AS other_chan_qty#57, (coalesce(ws_wc#32, 0.00) + coalesce(cs_wc#51, 0.00)) AS other_chan_wholesale_cost#58, (coalesce(ws_sp#33, 0.00) + coalesce(cs_sp#52, 0.00)) AS other_chan_sales_price#59, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(48) TakeOrderedAndProject -Input [13]: [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Arguments: 100, [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#57 ASC NULLS FIRST, other_chan_wholesale_cost#58 ASC NULLS FIRST, other_chan_sales_price#59 ASC NULLS FIRST, ratio#53 ASC NULLS FIRST], [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59] +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] + +(37) CometHashAggregate +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] + +(38) CometExchange +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometHashAggregate +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] + +(40) CometFilter +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) + +(41) CometSort +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] + +(42) CometSortMergeJoin +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner + +(43) CometProject +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] + +(44) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(45) CometFilter +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) + +(46) CometExchange +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(47) CometSort +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] + +(48) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(49) CometFilter +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) + +(50) CometProject +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] + +(51) CometExchange +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(52) CometSort +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] + +(53) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter + +(54) CometFilter +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) + +(55) CometProject +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] + +(56) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#50, d_year#51] + +(57) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight + +(58) CometProject +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] + +(59) CometHashAggregate +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] + +(60) CometExchange +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(61) CometHashAggregate +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] + +(62) CometFilter +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) + +(63) CometSort +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] + +(64) CometSortMergeJoin +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner + +(65) CometColumnarToRow [codegen id : 1] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(66) Project [codegen id : 1] +Output [13]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(67) TakeOrderedAndProject +Input [13]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: 100, [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, ratio#60 ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/extended.txt new file mode 100644 index 0000000000..f180fb1878 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/extended.txt @@ -0,0 +1,73 @@ + TakeOrderedAndProject [COMET: ] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding] + +- CometColumnarToRow + +- CometSortMergeJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 68 out of 70 eligible operators (97%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/simplified.txt index ebcb3c8230..5b555777a1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/simplified.txt @@ -1,15 +1,15 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] - ColumnarToRow + CometColumnarToRow InputAdapter CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] - CometHashAggregate [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum,sum,sum,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 - CometHashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum,ss_quantity,ss_wholesale_cost,ss_sales_price] + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] @@ -18,33 +18,52 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometHashAggregate [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum,sum,sum,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - ReusedExchange [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] #1 + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometHashAggregate [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum,sum,sum,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #5 - CometHashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum,cs_quantity,cs_wholesale_cost,cs_sales_price] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #6 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number] #3 + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/explain.txt index 6f7632050f..a10447b86c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/explain.txt @@ -1,93 +1,129 @@ == Physical Plan == -TakeOrderedAndProject (84) -+- * HashAggregate (83) - +- Exchange (82) - +- * HashAggregate (81) - +- Union (80) - :- * HashAggregate (69) - : +- Exchange (68) - : +- * HashAggregate (67) - : +- Union (66) - : :- * HashAggregate (38) - : : +- * ColumnarToRow (37) - : : +- CometExchange (36) - : : +- CometHashAggregate (35) - : : +- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometProject (28) - : : : +- CometBroadcastHashJoin (27) - : : : :- CometProject (22) - : : : : +- CometBroadcastHashJoin (21) - : : : : :- CometProject (17) - : : : : : +- CometBroadcastHashJoin (16) - : : : : : :- CometProject (11) - : : : : : : +- CometSortMergeJoin (10) - : : : : : : :- CometSort (4) - : : : : : : : +- CometExchange (3) - : : : : : : : +- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometSort (9) - : : : : : : +- CometExchange (8) - : : : : : : +- CometProject (7) - : : : : : : +- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : : : : +- CometBroadcastExchange (15) - : : : : : +- CometProject (14) - : : : : : +- CometFilter (13) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - : : : : +- CometBroadcastExchange (20) - : : : : +- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (18) - : : : +- CometBroadcastExchange (26) - : : : +- CometProject (25) - : : : +- CometFilter (24) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (23) - : : +- CometBroadcastExchange (32) - : : +- CometProject (31) - : : +- CometFilter (30) - : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (29) - : :- * HashAggregate (62) - : : +- * ColumnarToRow (61) - : : +- CometExchange (60) - : : +- CometHashAggregate (59) - : : +- CometProject (58) - : : +- CometBroadcastHashJoin (57) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometBroadcastHashJoin (51) - : : : : :- CometProject (49) - : : : : : +- CometBroadcastHashJoin (48) - : : : : : :- CometProject (46) - : : : : : : +- CometSortMergeJoin (45) - : : : : : : :- CometSort (42) - : : : : : : : +- CometExchange (41) - : : : : : : : +- CometFilter (40) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : : +- CometSort (44) - : : : : : : +- ReusedExchange (43) - : : : : : +- ReusedExchange (47) - : : : : +- ReusedExchange (50) - : : : +- ReusedExchange (53) - : : +- ReusedExchange (56) - : +- * HashAggregate (65) - : +- * ColumnarToRow (64) - : +- ReusedExchange (63) - :- * HashAggregate (74) - : +- Exchange (73) - : +- * HashAggregate (72) - : +- * HashAggregate (71) - : +- ReusedExchange (70) - +- * HashAggregate (79) - +- Exchange (78) - +- * HashAggregate (77) - +- * HashAggregate (76) - +- ReusedExchange (75) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` +* CometColumnarToRow (116) ++- CometTakeOrderedAndProject (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometHashAggregate (100) + : +- CometExchange (99) + : +- CometHashAggregate (98) + : +- CometUnion (97) + : :- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (29) + : : : +- CometBroadcastHashJoin (28) + : : : :- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometProject (17) + : : : : : +- CometBroadcastHashJoin (16) + : : : : : :- CometProject (11) + : : : : : : +- CometSortMergeJoin (10) + : : : : : : :- CometSort (4) + : : : : : : : +- CometExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometSort (9) + : : : : : : +- CometExchange (8) + : : : : : : +- CometProject (7) + : : : : : : +- CometFilter (6) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : : : : +- CometBroadcastExchange (15) + : : : : : +- CometProject (14) + : : : : : +- CometFilter (13) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.store (18) + : : : +- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.item (24) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometNativeScan parquet spark_catalog.default.promotion (30) + : :- CometHashAggregate (67) + : : +- CometExchange (66) + : : +- CometHashAggregate (65) + : : +- CometProject (64) + : : +- CometBroadcastHashJoin (63) + : : :- CometProject (61) + : : : +- CometBroadcastHashJoin (60) + : : : :- CometProject (58) + : : : : +- CometBroadcastHashJoin (57) + : : : : :- CometProject (52) + : : : : : +- CometBroadcastHashJoin (51) + : : : : : :- CometProject (49) + : : : : : : +- CometSortMergeJoin (48) + : : : : : : :- CometSort (42) + : : : : : : : +- CometExchange (41) + : : : : : : : +- CometFilter (40) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (39) + : : : : : : +- CometSort (47) + : : : : : : +- CometExchange (46) + : : : : : : +- CometProject (45) + : : : : : : +- CometFilter (44) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (43) + : : : : : +- ReusedExchange (50) + : : : : +- CometBroadcastExchange (56) + : : : : +- CometProject (55) + : : : : +- CometFilter (54) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (53) + : : : +- ReusedExchange (59) + : : +- ReusedExchange (62) + : +- CometHashAggregate (96) + : +- CometExchange (95) + : +- CometHashAggregate (94) + : +- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometProject (81) + : : : : +- CometBroadcastHashJoin (80) + : : : : :- CometProject (78) + : : : : : +- CometSortMergeJoin (77) + : : : : : :- CometSort (71) + : : : : : : +- CometExchange (70) + : : : : : : +- CometFilter (69) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (68) + : : : : : +- CometSort (76) + : : : : : +- CometExchange (75) + : : : : : +- CometProject (74) + : : : : : +- CometFilter (73) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (72) + : : : : +- ReusedExchange (79) + : : : +- CometBroadcastExchange (85) + : : : +- CometProject (84) + : : : +- CometFilter (83) + : : : +- CometNativeScan parquet spark_catalog.default.web_site (82) + : : +- ReusedExchange (88) + : +- ReusedExchange (91) + :- CometHashAggregate (105) + : +- CometExchange (104) + : +- CometHashAggregate (103) + : +- CometHashAggregate (102) + : +- ReusedExchange (101) + +- CometHashAggregate (110) + +- CometExchange (109) + +- CometHashAggregate (108) + +- CometHashAggregate (107) + +- ReusedExchange (106) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct (2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -101,9 +137,12 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct (6) CometFilter Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] @@ -130,9 +169,12 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(12) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] +ReadSchema: struct (13) CometFilter Input [2]: [d_date_sk#13, d_date#14] @@ -155,305 +197,450 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(18) CometNativeScan: `spark_catalog`.`default`.`store` +(18) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct (19) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(20) CometBroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] -(21) CometBroadcastHashJoin +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] +Right output [2]: [s_store_sk#15, s_store_id#17] Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(23) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17, i_current_price#18] - -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] - -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] - -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(29) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19, p_channel_tv#20] - -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] - -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] - -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight - -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(35) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) + +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] + +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] + +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(30) CometNativeScan parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) + +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] + +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] + +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight + +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -(36) CometExchange -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] - -(38) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Keys [1]: [s_store_id#16] +(38) CometHashAggregate +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#26, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28] -Results [5]: [store channel AS channel#29, concat(store, s_store_id#16) AS id#30, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#26,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28 AS profit#33] -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] +(39) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct (40) CometFilter -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Condition : ((isnotnull(cs_catalog_page_sk#34) AND isnotnull(cs_item_sk#35)) AND isnotnull(cs_promo_sk#36)) +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : ((isnotnull(cs_catalog_page_sk#27) AND isnotnull(cs_item_sk#28)) AND isnotnull(cs_promo_sk#29)) (41) CometExchange -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_item_sk#35, cs_order_number#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: hashpartitioning(cs_item_sk#28, cs_order_number#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (42) CometSort -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40], [cs_item_sk#35 ASC NULLS FIRST, cs_order_number#37 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33], [cs_item_sk#28 ASC NULLS FIRST, cs_order_number#30 ASC NULLS FIRST] -(43) ReusedExchange [Reuses operator id: 8] -Output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] +(43) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct -(44) CometSort -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST] +(44) CometFilter +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Condition : (isnotnull(cr_item_sk#34) AND isnotnull(cr_order_number#35)) -(45) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Right output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_item_sk#35, cs_order_number#37], [cr_item_sk#41, cr_order_number#42], LeftOuter +(45) CometProject +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] -(46) CometProject -Input [11]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] +(46) CometExchange +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: hashpartitioning(cr_item_sk#34, cr_order_number#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(47) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#45] +(47) CometSort +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34 ASC NULLS FIRST, cr_order_number#35 ASC NULLS FIRST] -(48) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] -Right output [1]: [d_date_sk#45] -Arguments: [cs_sold_date_sk#40], [d_date_sk#45], Inner, BuildRight +(48) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_item_sk#28, cs_order_number#30], [cr_item_sk#34, cr_order_number#35], LeftOuter (49) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44, d_date_sk#45] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] +Input [11]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] -(50) ReusedExchange [Reuses operator id: 20] -Output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] +(50) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#39] (51) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] -Right output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_catalog_page_sk#34], [cp_catalog_page_sk#46], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] +Right output [1]: [d_date_sk#39] +Arguments: [cs_sold_date_sk#33], [d_date_sk#39], Inner, BuildRight (52) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37, d_date_sk#39] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] -(53) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#48] +(53) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct -(54) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [i_item_sk#48] -Arguments: [cs_item_sk#35], [i_item_sk#48], Inner, BuildRight +(54) CometFilter +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Condition : isnotnull(cp_catalog_page_sk#40) (55) CometProject -Input [8]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, i_item_sk#48] -Arguments: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42], [cp_catalog_page_sk#40, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#41, 16, true, false, true) AS cp_catalog_page_id#42] -(56) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#49] +(56) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42] (57) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [p_promo_sk#49] -Arguments: [cs_promo_sk#36], [p_promo_sk#49], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] +Right output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_catalog_page_sk#27], [cp_catalog_page_sk#40], Inner, BuildRight (58) CometProject -Input [7]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, p_promo_sk#49] -Arguments: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(59) CometHashAggregate -Input [5]: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#38)), partial_sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] - -(60) CometExchange -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Arguments: hashpartitioning(cp_catalog_page_id#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(61) ColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] - -(62) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#38)), sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#38))#55, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57] -Results [5]: [catalog channel AS channel#58, concat(catalog_page, cp_catalog_page_id#47) AS id#59, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#38))#55,17,2) AS sales#60, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56 AS returns#61, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57 AS profit#62] - -(63) ReusedExchange [Reuses operator id: 36] -Output [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(64) ColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(65) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [1]: [web_site_id#63] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#69)), sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#69))#73, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75] -Results [5]: [web channel AS channel#76, concat(web_site, web_site_id#63) AS id#77, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#69))#73,17,2) AS sales#78, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74 AS returns#79, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75 AS profit#80] - -(66) Union - -(67) HashAggregate [codegen id : 4] -Input [5]: [channel#29, id#30, sales#31, returns#32, profit#33] -Keys [2]: [channel#29, id#30] -Functions [3]: [partial_sum(sales#31), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86] -Results [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] - -(68) Exchange -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Arguments: hashpartitioning(channel#29, id#30, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(69) HashAggregate [codegen id : 5] -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#93, sum(returns#32)#94, sum(profit#33)#95] -Results [5]: [channel#29, id#30, cast(sum(sales#31)#93 as decimal(37,2)) AS sales#96, cast(sum(returns#32)#94 as decimal(38,2)) AS returns#97, cast(sum(profit#33)#95 as decimal(38,2)) AS profit#98] - -(70) ReusedExchange [Reuses operator id: 68] -Output [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] - -(71) HashAggregate [codegen id : 10] -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#93, sum(returns#32)#94, sum(profit#33)#95] -Results [4]: [channel#29, sum(sales#31)#93 AS sales#99, sum(returns#32)#94 AS returns#100, sum(profit#33)#95 AS profit#101] - -(72) HashAggregate [codegen id : 10] -Input [4]: [channel#29, sales#99, returns#100, profit#101] -Keys [1]: [channel#29] -Functions [3]: [partial_sum(sales#99), partial_sum(returns#100), partial_sum(profit#101)] -Aggregate Attributes [6]: [sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] -Results [7]: [channel#29, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] - -(73) Exchange -Input [7]: [channel#29, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] -Arguments: hashpartitioning(channel#29, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(74) HashAggregate [codegen id : 11] -Input [7]: [channel#29, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] -Keys [1]: [channel#29] -Functions [3]: [sum(sales#99), sum(returns#100), sum(profit#101)] -Aggregate Attributes [3]: [sum(sales#99)#114, sum(returns#100)#115, sum(profit#101)#116] -Results [5]: [channel#29, null AS id#117, sum(sales#99)#114 AS sales#118, sum(returns#100)#115 AS returns#119, sum(profit#101)#116 AS profit#120] - -(75) ReusedExchange [Reuses operator id: 68] -Output [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] - -(76) HashAggregate [codegen id : 16] -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#93, sum(returns#32)#94, sum(profit#33)#95] -Results [3]: [sum(sales#31)#93 AS sales#121, sum(returns#32)#94 AS returns#122, sum(profit#33)#95 AS profit#123] - -(77) HashAggregate [codegen id : 16] -Input [3]: [sales#121, returns#122, profit#123] +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(59) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#43] + +(60) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [i_item_sk#43] +Arguments: [cs_item_sk#28], [i_item_sk#43], Inner, BuildRight + +(61) CometProject +Input [8]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, i_item_sk#43] +Arguments: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(62) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#44] + +(63) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [p_promo_sk#44] +Arguments: [cs_promo_sk#29], [p_promo_sk#44], Inner, BuildRight + +(64) CometProject +Input [7]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, p_promo_sk#44] +Arguments: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(65) CometHashAggregate +Input [5]: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(66) CometExchange +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Arguments: hashpartitioning(cp_catalog_page_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(67) CometHashAggregate +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(68) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#56)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(69) CometFilter +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Condition : ((isnotnull(ws_web_site_sk#51) AND isnotnull(ws_item_sk#50)) AND isnotnull(ws_promo_sk#52)) + +(70) CometExchange +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: hashpartitioning(ws_item_sk#50, ws_order_number#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(71) CometSort +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56], [ws_item_sk#50 ASC NULLS FIRST, ws_order_number#53 ASC NULLS FIRST] + +(72) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] +ReadSchema: struct + +(73) CometFilter +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Condition : (isnotnull(wr_item_sk#57) AND isnotnull(wr_order_number#58)) + +(74) CometProject +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] + +(75) CometExchange +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: hashpartitioning(wr_item_sk#57, wr_order_number#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(76) CometSort +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57 ASC NULLS FIRST, wr_order_number#58 ASC NULLS FIRST] + +(77) CometSortMergeJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Right output [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_order_number#53], [wr_item_sk#57, wr_order_number#58], LeftOuter + +(78) CometProject +Input [11]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] + +(79) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#62] + +(80) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] +Right output [1]: [d_date_sk#62] +Arguments: [ws_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(81) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60, d_date_sk#62] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] + +(82) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#63, web_site_id#64] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(83) CometFilter +Input [2]: [web_site_sk#63, web_site_id#64] +Condition : isnotnull(web_site_sk#63) + +(84) CometProject +Input [2]: [web_site_sk#63, web_site_id#64] +Arguments: [web_site_sk#63, web_site_id#65], [web_site_sk#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#64, 16, true, false, true) AS web_site_id#65] + +(85) CometBroadcastExchange +Input [2]: [web_site_sk#63, web_site_id#65] +Arguments: [web_site_sk#63, web_site_id#65] + +(86) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] +Right output [2]: [web_site_sk#63, web_site_id#65] +Arguments: [ws_web_site_sk#51], [web_site_sk#63], Inner, BuildRight + +(87) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_sk#63, web_site_id#65] +Arguments: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(88) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#66] + +(89) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [i_item_sk#66] +Arguments: [ws_item_sk#50], [i_item_sk#66], Inner, BuildRight + +(90) CometProject +Input [8]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, i_item_sk#66] +Arguments: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(91) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#67] + +(92) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [p_promo_sk#67] +Arguments: [ws_promo_sk#52], [p_promo_sk#67], Inner, BuildRight + +(93) CometProject +Input [7]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, p_promo_sk#67] +Arguments: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(94) CometHashAggregate +Input [5]: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Keys [1]: [web_site_id#65] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#54)), partial_sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(95) CometExchange +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Arguments: hashpartitioning(web_site_id#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(96) CometHashAggregate +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Keys [1]: [web_site_id#65] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#54)), sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(97) CometUnion +Child 0 Input [5]: [channel#73, id#74, sales#75, returns#76, profit#77] +Child 1 Input [5]: [channel#78, id#79, sales#80, returns#81, profit#82] +Child 2 Input [5]: [channel#83, id#84, sales#85, returns#86, profit#87] + +(98) CometHashAggregate +Input [5]: [channel#73, id#74, sales#75, returns#76, profit#77] +Keys [2]: [channel#73, id#74] +Functions [3]: [partial_sum(sales#75), partial_sum(returns#76), partial_sum(profit#77)] + +(99) CometExchange +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Arguments: hashpartitioning(channel#73, id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(100) CometHashAggregate +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#73, id#74] +Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] + +(101) ReusedExchange [Reuses operator id: 99] +Output [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(102) CometHashAggregate +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#73, id#74] +Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] + +(103) CometHashAggregate +Input [4]: [channel#73, sales#94, returns#95, profit#96] +Keys [1]: [channel#73] +Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] + +(104) CometExchange +Input [7]: [channel#73, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Arguments: hashpartitioning(channel#73, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(105) CometHashAggregate +Input [7]: [channel#73, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Keys [1]: [channel#73] +Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] + +(106) ReusedExchange [Reuses operator id: 99] +Output [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(107) CometHashAggregate +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#73, id#74] +Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] + +(108) CometHashAggregate +Input [3]: [sales#103, returns#104, profit#105] Keys: [] -Functions [3]: [partial_sum(sales#121), partial_sum(returns#122), partial_sum(profit#123)] -Aggregate Attributes [6]: [sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Results [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +Functions [3]: [partial_sum(sales#103), partial_sum(returns#104), partial_sum(profit#105)] -(78) Exchange -Input [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(109) CometExchange +Input [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(79) HashAggregate [codegen id : 17] -Input [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +(110) CometHashAggregate +Input [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] Keys: [] -Functions [3]: [sum(sales#121), sum(returns#122), sum(profit#123)] -Aggregate Attributes [3]: [sum(sales#121)#136, sum(returns#122)#137, sum(profit#123)#138] -Results [5]: [null AS channel#139, null AS id#140, sum(sales#121)#136 AS sales#141, sum(returns#122)#137 AS returns#142, sum(profit#123)#138 AS profit#143] +Functions [3]: [sum(sales#103), sum(returns#104), sum(profit#105)] -(80) Union +(111) CometUnion +Child 0 Input [5]: [channel#73, id#74, sales#112, returns#113, profit#114] +Child 1 Input [5]: [channel#73, id#115, sales#116, returns#117, profit#118] +Child 2 Input [5]: [channel#119, id#120, sales#121, returns#122, profit#123] -(81) HashAggregate [codegen id : 18] -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Keys [5]: [channel#29, id#30, sales#96, returns#97, profit#98] +(112) CometHashAggregate +Input [5]: [channel#73, id#74, sales#112, returns#113, profit#114] +Keys [5]: [channel#73, id#74, sales#112, returns#113, profit#114] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -(82) Exchange -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Arguments: hashpartitioning(channel#29, id#30, sales#96, returns#97, profit#98, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(113) CometExchange +Input [5]: [channel#73, id#74, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#73, id#74, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(83) HashAggregate [codegen id : 19] -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Keys [5]: [channel#29, id#30, sales#96, returns#97, profit#98] +(114) CometHashAggregate +Input [5]: [channel#73, id#74, sales#112, returns#113, profit#114] +Keys [5]: [channel#73, id#74, sales#112, returns#113, profit#114] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -(84) TakeOrderedAndProject -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Arguments: 100, [channel#29 ASC NULLS FIRST, id#30 ASC NULLS FIRST], [channel#29, id#30, sales#96, returns#97, profit#98] +(115) CometTakeOrderedAndProject +Input [5]: [channel#73, id#74, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#73 ASC NULLS FIRST,id#74 ASC NULLS FIRST], output=[channel#73,id#74,sales#112,returns#113,profit#114]), [channel#73, id#74, sales#112, returns#113, profit#114], 100, 0, [channel#73 ASC NULLS FIRST, id#74 ASC NULLS FIRST], [channel#73, id#74, sales#112, returns#113, profit#114] + +(116) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#73, id#74, sales#112, returns#113, profit#114] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/extended.txt new file mode 100644 index 0000000000..9f0aaa8522 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/extended.txt @@ -0,0 +1,368 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_site + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_site + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_site + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 365 out of 365 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/simplified.txt index 4f7de727bb..4bdc3ca774 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/simplified.txt @@ -1,106 +1,118 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (19) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #4 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #5 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #10 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #11 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #5 - ReusedExchange [d_date_sk] #6 - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [web_site_id,sum,sum,isEmpty,sum,isEmpty] #3 - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #12 - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #13 - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometExchange [channel,id,sales,returns,profit] #1 + CometHashAggregate [channel,id,sales,returns,profit] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id] #2 + CometHashAggregate [sales,returns,profit] [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,s_store_id,sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00)))] + CometExchange [s_store_id] #3 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,cp_catalog_page_id,sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00)))] + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,web_site_id,sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00)))] + CometExchange [web_site_id] #14 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel] #18 + CometHashAggregate [sales,returns,profit] [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange #19 + CometHashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/explain.txt index 4fe52ded5c..d2f5f6f1f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/explain.txt @@ -1,53 +1,61 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * Project (35) - +- Window (34) - +- * ColumnarToRow (33) - +- CometSort (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometUnion (27) - :- CometHashAggregate (16) - : +- CometExchange (15) - : +- CometHashAggregate (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (37) ++- * Project (36) + +- Window (35) + +- * CometColumnarToRow (34) + +- CometSort (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometUnion (28) + :- CometHashAggregate (17) + : +- CometExchange (16) + : +- CometHashAggregate (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - :- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometHashAggregate (23) - +- ReusedExchange (22) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + :- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometHashAggregate (19) + : +- ReusedExchange (18) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometHashAggregate (24) + +- ReusedExchange (23) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct (4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] @@ -70,124 +78,131 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometNativeScan: `spark_catalog`.`default`.`item` +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (10) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_class#7, i_category#8], [ws_net_paid#2, i_class#7, i_category#8] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] -(14) CometHashAggregate -Input [3]: [ws_net_paid#2, i_class#7, i_category#8] -Keys [2]: [i_category#8, i_class#7] +(15) CometHashAggregate +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(15) CometExchange -Input [3]: [i_category#8, i_class#7, sum#9] -Arguments: hashpartitioning(i_category#8, i_class#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [i_category#8, i_class#7, sum#9] -Keys [2]: [i_category#8, i_class#7] +(17) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [i_category#10, i_class#11, sum#12] +(18) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#12] -(18) CometHashAggregate -Input [3]: [i_category#10, i_class#11, sum#12] -Keys [2]: [i_category#10, i_class#11] +(19) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#12] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#13))] -(19) CometHashAggregate +(20) CometHashAggregate Input [2]: [total_sum#14, i_category#10] Keys [1]: [i_category#10] Functions [1]: [partial_sum(total_sum#14)] -(20) CometExchange +(21) CometExchange Input [3]: [i_category#10, sum#15, isEmpty#16] Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(21) CometHashAggregate +(22) CometHashAggregate Input [3]: [i_category#10, sum#15, isEmpty#16] Keys [1]: [i_category#10] Functions [1]: [sum(total_sum#14)] -(22) ReusedExchange [Reuses operator id: 15] -Output [3]: [i_category#17, i_class#18, sum#19] - -(23) CometHashAggregate -Input [3]: [i_category#17, i_class#18, sum#19] -Keys [2]: [i_category#17, i_class#18] -Functions [1]: [sum(UnscaledValue(ws_net_paid#20))] +(23) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#17] (24) CometHashAggregate -Input [1]: [total_sum#21] +Input [3]: [i_category#10, i_class#9, sum#17] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#18))] + +(25) CometHashAggregate +Input [1]: [total_sum#19] Keys: [] -Functions [1]: [partial_sum(total_sum#21)] +Functions [1]: [partial_sum(total_sum#19)] -(25) CometExchange -Input [2]: [sum#22, isEmpty#23] +(26) CometExchange +Input [2]: [sum#20, isEmpty#21] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(26) CometHashAggregate -Input [2]: [sum#22, isEmpty#23] +(27) CometHashAggregate +Input [2]: [sum#20, isEmpty#21] Keys: [] -Functions [1]: [sum(total_sum#21)] +Functions [1]: [sum(total_sum#19)] -(27) CometUnion -Child 0 Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] +(28) CometUnion +Child 0 Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Child 1 Input [6]: [total_sum#28, i_category#10, i_class#29, g_category#30, g_class#31, lochierarchy#32] Child 2 Input [6]: [total_sum#33, i_category#34, i_class#35, g_category#36, g_class#37, lochierarchy#38] -(28) CometHashAggregate -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] +(29) CometHashAggregate +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Functions: [] -(29) CometExchange -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Arguments: hashpartitioning(total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(30) CometExchange +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Arguments: hashpartitioning(total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(30) CometHashAggregate -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] +(31) CometHashAggregate +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Functions: [] -(31) CometExchange -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] +(32) CometExchange +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] Arguments: hashpartitioning(lochierarchy#27, _w0#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(32) CometSort -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] -Arguments: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39], [lochierarchy#27 ASC NULLS FIRST, _w0#39 ASC NULLS FIRST, total_sum#24 DESC NULLS LAST] +(33) CometSort +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Arguments: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39], [lochierarchy#27 ASC NULLS FIRST, _w0#39 ASC NULLS FIRST, total_sum#22 DESC NULLS LAST] -(33) ColumnarToRow [codegen id : 1] -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] -(34) Window -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] -Arguments: [rank(total_sum#24) windowspecdefinition(lochierarchy#27, _w0#39, total_sum#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#40], [lochierarchy#27, _w0#39], [total_sum#24 DESC NULLS LAST] +(35) Window +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Arguments: [rank(total_sum#22) windowspecdefinition(lochierarchy#27, _w0#39, total_sum#22 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#40], [lochierarchy#27, _w0#39], [total_sum#22 DESC NULLS LAST] -(35) Project [codegen id : 2] -Output [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] -Input [6]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39, rank_within_parent#40] +(36) Project [codegen id : 2] +Output [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] +Input [6]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39, rank_within_parent#40] -(36) TakeOrderedAndProject -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#8 END ASC NULLS FIRST, rank_within_parent#40 ASC NULLS FIRST], [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] +(37) TakeOrderedAndProject +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] +Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#23 END ASC NULLS FIRST, rank_within_parent#40 ASC NULLS FIRST], [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/extended.txt new file mode 100644 index 0000000000..fa549246d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/extended.txt @@ -0,0 +1,69 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 63 out of 66 eligible operators (95%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/simplified.txt index 5803acd613..d86038290c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/simplified.txt @@ -4,37 +4,38 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter Window [total_sum,lochierarchy,_w0] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0] CometExchange [lochierarchy,_w0] #1 - CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,g_category,g_class] + CometHashAggregate [g_category] [total_sum,i_category,i_class,lochierarchy,_w0,g_class] CometExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] CometUnion [total_sum,i_category,i_class,g_category,g_class,lochierarchy] - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [sum] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(UnscaledValue(ws_net_paid)),i_category,i_class] CometExchange [i_category,i_class] #3 - CometHashAggregate [i_category,i_class,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [i_category,i_class,sum] CometProject [ws_net_paid,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] CometExchange [i_category] #6 - CometHashAggregate [i_category,sum,isEmpty,total_sum] - CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [total_sum] [i_category,sum,isEmpty] + CometHashAggregate [i_class,sum] [total_sum,i_category,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] CometExchange #7 - CometHashAggregate [sum,isEmpty,total_sum] - CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [total_sum] [sum,isEmpty] + CometHashAggregate [i_category,i_class,sum] [total_sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/explain.txt index b2dbca0416..1cfd35fde7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/explain.txt @@ -1,125 +1,140 @@ == Physical Plan == -* ColumnarToRow (24) -+- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (25) ++- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct (2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometNativeScan: `spark_catalog`.`default`.`item` +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(22) CometColumnarExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +(23) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(23) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST] +(24) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(24) ColumnarToRow [codegen id : 3] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(25) CometColumnarToRow [codegen id : 3] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/extended.txt new file mode 100644 index 0000000000..5bbef326d3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 21 out of 23 eligible operators (91%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/simplified.txt index ff36840a5f..7acc6e37c6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 @@ -8,23 +8,24 @@ WholeStageCodegen (3) InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] CometExchange [i_class] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/explain.txt new file mode 100644 index 0000000000..4362f5a927 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/explain.txt @@ -0,0 +1,230 @@ +== Physical Plan == +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) + + +(1) CometNativeScan parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) + +(3) CometNativeScan parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct + +(4) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) + +(6) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#4] +Arguments: [ss_customer_sk#4] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight + +(12) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct + +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#11] + +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customer_sk#12], [ws_bill_customer_sk#9 AS customer_sk#12] + +(16) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +ReadSchema: struct + +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#15] + +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight + +(19) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customer_sk#16], [cs_ship_customer_sk#13 AS customer_sk#16] + +(20) CometUnion +Child 0 Input [1]: [customer_sk#12] +Child 1 Input [1]: [customer_sk#16] + +(21) CometBroadcastExchange +Input [1]: [customer_sk#12] +Arguments: [customer_sk#12] + +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customer_sk#12] +Arguments: [c_customer_sk#1], [customer_sk#12], LeftSemi, BuildRight + +(23) CometProject +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] + +(24) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_county#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) + +(26) CometProject +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] + +(27) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] + +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight + +(29) CometProject +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] +Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] + +(30) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) + +(32) CometProject +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#20, 1)) AS cd_gender#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) AS cd_marital_status#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) AS cd_education_status#30, cd_purchase_estimate#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#24, 10)) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(33) CometBroadcastExchange +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(34) CometBroadcastHashJoin +Left output [1]: [c_current_cdemo_sk#2] +Right output [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight + +(35) CometProject +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(36) CometHashAggregate +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Functions [1]: [partial_count(1)] + +(37) CometExchange +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(38) CometHashAggregate +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Functions [1]: [count(1)] + +(39) CometTakeOrderedAndProject +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#28 ASC NULLS FIRST,cd_marital_status#29 ASC NULLS FIRST,cd_education_status#30 ASC NULLS FIRST,cd_purchase_estimate#23 ASC NULLS FIRST,cd_credit_rating#31 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[cd_gender#28,cd_marital_status#29,cd_education_status#30,cnt1#33,cd_purchase_estimate#23,cnt2#34,cd_credit_rating#31,cnt3#35,cd_dep_count#25,cnt4#36,cd_dep_employed_count#26,cnt5#37,cd_dep_college_count#27,cnt6#38]), [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38], 100, 0, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] + +(40) CometColumnarToRow [codegen id : 1] +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/extended.txt new file mode 100644 index 0000000000..42c478c7c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..837c58b9aa --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/simplified.txt @@ -0,0 +1,42 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] + CometHashAggregate [count] [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count(1)] + CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [customer_sk] #4 + CometUnion [customer_sk] + CometProject [ws_bill_customer_sk] [customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk] #5 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/explain.txt new file mode 100644 index 0000000000..2978c475bd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/explain.txt @@ -0,0 +1,406 @@ +== Physical Plan == +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometNativeScan parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) + + +(1) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] + +(17) HashAggregate [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22] +Results [2]: [c_customer_id#9 AS customer_id#23, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22,18,2) AS year_total#24] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#23, year_total#24] +Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) + +(19) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Condition : (isnotnull(c_customer_sk#25) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)))) + +(21) CometProject +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Arguments: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38], [c_customer_sk#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)) AS c_customer_id#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#29, 1)) AS c_preferred_cust_flag#36, c_birth_country#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#31, 13)) AS c_login#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#32, 50)) AS c_email_address#38] + +(22) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(23) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Right output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_sk#25], [ss_customer_sk#39], Inner, BuildRight + +(24) CometProject +Input [12]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(25) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] + +(28) CometBroadcastHashJoin +Left output [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#43], Inner, BuildRight + +(29) CometProject +Input [12]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] + +(30) CometHashAggregate +Input [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] + +(31) CometExchange +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] + +(33) HashAggregate [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22] +Results [5]: [c_customer_id#33 AS customer_id#46, c_first_name#34 AS customer_first_name#47, c_last_name#35 AS customer_last_name#48, c_email_address#38 AS customer_email_address#49, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22,18,2) AS year_total#50] + +(34) BroadcastExchange +Input [5]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#46] +Join type: Inner +Join condition: None + +(36) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(37) CometFilter +Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Condition : (isnotnull(c_customer_sk#51) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)))) + +(38) CometProject +Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Arguments: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64], [c_customer_sk#51, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)) AS c_customer_id#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#53, 20)) AS c_first_name#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#54, 30)) AS c_last_name#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#55, 1)) AS c_preferred_cust_flag#62, c_birth_country#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#57, 13)) AS c_login#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#58, 50)) AS c_email_address#64] + +(39) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#68)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(40) CometFilter +Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_bill_customer_sk#65) + +(41) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] + +(42) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64] +Right output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [c_customer_sk#51], [ws_bill_customer_sk#65], Inner, BuildRight + +(43) CometProject +Input [12]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] + +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#69, d_year#70] + +(45) CometBroadcastHashJoin +Left output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Right output [2]: [d_date_sk#69, d_year#70] +Arguments: [ws_sold_date_sk#68], [d_date_sk#69], Inner, BuildRight + +(46) CometProject +Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68, d_date_sk#69, d_year#70] +Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70] + +(47) CometHashAggregate +Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70] +Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] + +(48) CometExchange +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] +Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(49) CometColumnarToRow [codegen id : 2] +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] + +(50) HashAggregate [codegen id : 2] +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] +Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#72] +Results [2]: [c_customer_id#59 AS customer_id#73, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#72,18,2) AS year_total#74] + +(51) Filter [codegen id : 2] +Input [2]: [customer_id#73, year_total#74] +Condition : (isnotnull(year_total#74) AND (year_total#74 > 0.00)) + +(52) BroadcastExchange +Input [2]: [customer_id#73, year_total#74] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#73] +Join type: Inner +Join condition: None + +(54) Project [codegen id : 4] +Output [8]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, year_total#74] +Input [9]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, customer_id#73, year_total#74] + +(55) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(56) CometFilter +Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Condition : (isnotnull(c_customer_sk#75) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#76, 16)))) + +(57) CometProject +Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Arguments: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88], [c_customer_sk#75, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#76, 16)) AS c_customer_id#83, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#77, 20)) AS c_first_name#84, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#78, 30)) AS c_last_name#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#79, 1)) AS c_preferred_cust_flag#86, c_birth_country#80, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#81, 13)) AS c_login#87, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#82, 50)) AS c_email_address#88] + +(58) ReusedExchange [Reuses operator id: 41] +Output [4]: [ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] + +(59) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88] +Right output [4]: [ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Arguments: [c_customer_sk#75], [ws_bill_customer_sk#89], Inner, BuildRight + +(60) CometProject +Input [12]: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Arguments: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92], [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] + +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#93, d_year#94] + +(62) CometBroadcastHashJoin +Left output [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ws_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight + +(63) CometProject +Input [12]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94], [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94] + +(64) CometHashAggregate +Input [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))] + +(65) CometExchange +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] +Arguments: hashpartitioning(c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometColumnarToRow [codegen id : 3] +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] + +(67) HashAggregate [codegen id : 3] +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] +Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))#72] +Results [2]: [c_customer_id#83 AS customer_id#96, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))#72,18,2) AS year_total#97] + +(68) BroadcastExchange +Input [2]: [customer_id#96, year_total#97] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#96] +Join type: Inner +Join condition: (CASE WHEN (year_total#74 > 0.00) THEN (year_total#97 / year_total#74) ELSE 0E-20 END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#50 / year_total#24) ELSE 0E-20 END) + +(70) Project [codegen id : 4] +Output [4]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] +Input [10]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, year_total#74, customer_id#96, year_total#97] + +(71) TakeOrderedAndProject +Input [4]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] +Arguments: 100, [customer_id#46 ASC NULLS FIRST, customer_first_name#47 ASC NULLS FIRST, customer_last_name#48 ASC NULLS FIRST, customer_email_address#49 ASC NULLS FIRST], [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/extended.txt new file mode 100644 index 0000000000..e6c4a8fc34 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/extended.txt @@ -0,0 +1,81 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- BroadcastHashJoin + : : :- Filter + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#1 AS customer_id#2, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#3 - ss_ext_discount_amt#4)))#5,18,2) AS year_total#6)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#7 AS customer_id#8, c_first_name#9 AS customer_first_name#10, c_last_name#11 AS customer_last_name#12, c_email_address#13 AS customer_email_address#14, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#15 - ss_ext_discount_amt#16)))#5,18,2) AS year_total#17)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#18 AS customer_id#19, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#20 - ws_ext_discount_amt#21)))#22,18,2) AS year_total#23)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#24 AS customer_id#25, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#26 - ws_ext_discount_amt#27)))#22,18,2) AS year_total#28)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 60 out of 75 eligible operators (80%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/simplified.txt new file mode 100644 index 0000000000..556f81da08 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_datafusion/simplified.txt @@ -0,0 +1,82 @@ +TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] + WholeStageCodegen (4) + Project [customer_id,customer_first_name,customer_last_name,customer_email_address] + BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] + Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] + BroadcastHashJoin [customer_id,customer_id] + BroadcastHashJoin [customer_id,customer_id] + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/explain.txt new file mode 100644 index 0000000000..e9c6b13b83 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/explain.txt @@ -0,0 +1,137 @@ +== Physical Plan == +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS _w0#18] + +(19) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] + +(22) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] + +(24) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/extended.txt new file mode 100644 index 0000000000..5c9a0c0c63 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/extended.txt @@ -0,0 +1,26 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#1, i_item_desc#2, i_category#3, i_class#4, i_current_price#5, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#6))#7,17,2) AS itemrevenue#8, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#6))#7,17,2) AS _w0#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 18 out of 22 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/simplified.txt new file mode 100644 index 0000000000..12828df0ce --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_datafusion/simplified.txt @@ -0,0 +1,30 @@ +TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] + WholeStageCodegen (3) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/explain.txt new file mode 100644 index 0000000000..7bf6898973 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/explain.txt @@ -0,0 +1,812 @@ +== Physical Plan == +* CometColumnarToRow (122) ++- CometTakeOrderedAndProject (121) + +- CometHashAggregate (120) + +- CometExchange (119) + +- CometHashAggregate (118) + +- CometUnion (117) + :- CometHashAggregate (96) + : +- CometExchange (95) + : +- CometHashAggregate (94) + : +- CometUnion (93) + : :- CometFilter (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometBroadcastHashJoin (47) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (46) + : : : : +- CometProject (45) + : : : : +- CometBroadcastHashJoin (44) + : : : : :- CometFilter (4) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : : +- CometBroadcastExchange (43) + : : : : +- CometBroadcastHashJoin (42) + : : : : :- CometHashAggregate (32) + : : : : : +- CometExchange (31) + : : : : : +- CometHashAggregate (30) + : : : : : +- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (26) + : : : : : : +- CometBroadcastHashJoin (25) + : : : : : : :- CometFilter (6) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometBroadcastExchange (24) + : : : : : : +- CometBroadcastHashJoin (23) + : : : : : : :- CometFilter (8) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) + : : : : : : +- CometBroadcastExchange (22) + : : : : : : +- CometProject (21) + : : : : : : +- CometBroadcastHashJoin (20) + : : : : : : :- CometProject (15) + : : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : : :- CometFilter (10) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometBroadcastExchange (13) + : : : : : : : +- CometFilter (12) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : : : : +- CometBroadcastExchange (19) + : : : : : : +- CometProject (18) + : : : : : : +- CometFilter (17) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) + : : : : : +- ReusedExchange (27) + : : : : +- CometBroadcastExchange (41) + : : : : +- CometProject (40) + : : : : +- CometBroadcastHashJoin (39) + : : : : :- CometProject (37) + : : : : : +- CometBroadcastHashJoin (36) + : : : : : :- CometFilter (34) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : : +- ReusedExchange (35) + : : : : +- ReusedExchange (38) + : : : +- CometBroadcastExchange (52) + : : : +- CometBroadcastHashJoin (51) + : : : :- CometFilter (49) + : : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : : +- ReusedExchange (50) + : : +- CometBroadcastExchange (58) + : : +- CometProject (57) + : : +- CometFilter (56) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + : :- CometFilter (78) + : : +- CometHashAggregate (77) + : : +- CometExchange (76) + : : +- CometHashAggregate (75) + : : +- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometBroadcastHashJoin (68) + : : : : :- CometFilter (66) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (65) + : : : : +- ReusedExchange (67) + : : : +- ReusedExchange (69) + : : +- ReusedExchange (72) + : +- CometFilter (92) + : +- CometHashAggregate (91) + : +- CometExchange (90) + : +- CometHashAggregate (89) + : +- CometProject (88) + : +- CometBroadcastHashJoin (87) + : :- CometProject (85) + : : +- CometBroadcastHashJoin (84) + : : :- CometBroadcastHashJoin (82) + : : : :- CometFilter (80) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (79) + : : : +- ReusedExchange (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (86) + :- CometHashAggregate (101) + : +- CometExchange (100) + : +- CometHashAggregate (99) + : +- CometHashAggregate (98) + : +- ReusedExchange (97) + :- CometHashAggregate (106) + : +- CometExchange (105) + : +- CometHashAggregate (104) + : +- CometHashAggregate (103) + : +- ReusedExchange (102) + :- CometHashAggregate (111) + : +- CometExchange (110) + : +- CometHashAggregate (109) + : +- CometHashAggregate (108) + : +- ReusedExchange (107) + +- CometHashAggregate (116) + +- CometExchange (115) + +- CometHashAggregate (114) + +- CometHashAggregate (113) + +- ReusedExchange (112) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) + +(5) CometNativeScan parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) + +(7) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(8) CometFilter +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) + +(9) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) + +(11) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(12) CometFilter +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) + +(13) CometBroadcastExchange +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] + +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight + +(15) CometProject +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] + +(16) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) + +(18) CometProject +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(19) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight + +(21) CometProject +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] + +(22) CometBroadcastExchange +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] + +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight + +(24) CometBroadcastExchange +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] + +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight + +(26) CometProject +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] + +(27) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#23] + +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight + +(29) CometProject +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] + +(30) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(31) CometExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(32) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(33) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight + +(43) CometBroadcastExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] + +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight + +(45) CometProject +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] + +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] + +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight + +(48) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) + +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] + +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight + +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] + +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight + +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] + +(55) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) + +(57) CometProject +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] + +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] + +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] + +(64) CometFilter +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(65) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(66) CometFilter +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) + +(67) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#54] + +(68) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#54] +Arguments: [cs_item_sk#50], [ss_item_sk#54], LeftSemi, BuildRight + +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] + +(70) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_item_sk#50], [i_item_sk#55], Inner, BuildRight + +(71) CometProject +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] + +(72) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#59] + +(73) CometBroadcastHashJoin +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [cs_sold_date_sk#53], [d_date_sk#59], Inner, BuildRight + +(74) CometProject +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] + +(75) CometHashAggregate +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] + +(76) CometExchange +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(77) CometHashAggregate +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] + +(78) CometFilter +Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(79) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#69)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(80) CometFilter +Input [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Condition : isnotnull(ws_item_sk#66) + +(81) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#70] + +(82) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [1]: [ss_item_sk#70] +Arguments: [ws_item_sk#66], [ss_item_sk#70], LeftSemi, BuildRight + +(83) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_item_sk#66], [i_item_sk#71], Inner, BuildRight + +(85) CometProject +Input [8]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] + +(86) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#75] + +(87) CometBroadcastHashJoin +Left output [6]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] +Right output [1]: [d_date_sk#75] +Arguments: [ws_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight + +(88) CometProject +Input [7]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74, d_date_sk#75] +Arguments: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] + +(89) CometHashAggregate +Input [5]: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [partial_sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), partial_count(1)] + +(90) CometExchange +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Arguments: hashpartitioning(i_brand_id#72, i_class_id#73, i_category_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(91) CometHashAggregate +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), count(1)] + +(92) CometFilter +Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(93) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Child 2 Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] + +(94) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] + +(95) CometExchange +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(96) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(97) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum#89, isEmpty#90, sum#91] + +(98) CometHashAggregate +Input [7]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum#89, isEmpty#90, sum#91] +Keys [4]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88] +Functions [2]: [sum(sales#92), sum(number_sales#93)] + +(99) CometHashAggregate +Input [5]: [channel#85, i_brand_id#86, i_class_id#87, sum_sales#94, number_sales#95] +Keys [3]: [channel#85, i_brand_id#86, i_class_id#87] +Functions [2]: [partial_sum(sum_sales#94), partial_sum(number_sales#95)] + +(100) CometExchange +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, sum#96, isEmpty#97, sum#98] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(101) CometHashAggregate +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, sum#96, isEmpty#97, sum#98] +Keys [3]: [channel#85, i_brand_id#86, i_class_id#87] +Functions [2]: [sum(sum_sales#94), sum(number_sales#95)] + +(102) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102, sum#103, isEmpty#104, sum#105] + +(103) CometHashAggregate +Input [7]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102, sum#103, isEmpty#104, sum#105] +Keys [4]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102] +Functions [2]: [sum(sales#106), sum(number_sales#107)] + +(104) CometHashAggregate +Input [4]: [channel#99, i_brand_id#100, sum_sales#108, number_sales#109] +Keys [2]: [channel#99, i_brand_id#100] +Functions [2]: [partial_sum(sum_sales#108), partial_sum(number_sales#109)] + +(105) CometExchange +Input [5]: [channel#99, i_brand_id#100, sum#110, isEmpty#111, sum#112] +Arguments: hashpartitioning(channel#99, i_brand_id#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(106) CometHashAggregate +Input [5]: [channel#99, i_brand_id#100, sum#110, isEmpty#111, sum#112] +Keys [2]: [channel#99, i_brand_id#100] +Functions [2]: [sum(sum_sales#108), sum(number_sales#109)] + +(107) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] + +(108) CometHashAggregate +Input [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] +Keys [4]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116] +Functions [2]: [sum(sales#120), sum(number_sales#121)] + +(109) CometHashAggregate +Input [3]: [channel#113, sum_sales#122, number_sales#123] +Keys [1]: [channel#113] +Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)] + +(110) CometExchange +Input [4]: [channel#113, sum#124, isEmpty#125, sum#126] +Arguments: hashpartitioning(channel#113, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(111) CometHashAggregate +Input [4]: [channel#113, sum#124, isEmpty#125, sum#126] +Keys [1]: [channel#113] +Functions [2]: [sum(sum_sales#122), sum(number_sales#123)] + +(112) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130, sum#131, isEmpty#132, sum#133] + +(113) CometHashAggregate +Input [7]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130, sum#131, isEmpty#132, sum#133] +Keys [4]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130] +Functions [2]: [sum(sales#134), sum(number_sales#135)] + +(114) CometHashAggregate +Input [2]: [sum_sales#136, number_sales#137] +Keys: [] +Functions [2]: [partial_sum(sum_sales#136), partial_sum(number_sales#137)] + +(115) CometExchange +Input [3]: [sum#138, isEmpty#139, sum#140] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(116) CometHashAggregate +Input [3]: [sum#138, isEmpty#139, sum#140] +Keys: [] +Functions [2]: [sum(sum_sales#136), sum(number_sales#137)] + +(117) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Child 1 Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#143, sum(sum_sales)#144, sum(number_sales)#145] +Child 2 Input [6]: [channel#99, i_brand_id#100, i_class_id#146, i_category_id#147, sum(sum_sales)#148, sum(number_sales)#149] +Child 3 Input [6]: [channel#113, i_brand_id#150, i_class_id#151, i_category_id#152, sum(sum_sales)#153, sum(number_sales)#154] +Child 4 Input [6]: [channel#155, i_brand_id#156, i_class_id#157, i_category_id#158, sum(sum_sales)#159, sum(number_sales)#160] + +(118) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Functions: [] + +(119) CometExchange +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(120) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Functions: [] + +(121) CometTakeOrderedAndProject +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#141,number_sales#142]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] + +(122) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] +* CometColumnarToRow (142) ++- CometHashAggregate (141) + +- CometExchange (140) + +- CometHashAggregate (139) + +- CometUnion (138) + :- CometProject (126) + : +- CometBroadcastHashJoin (125) + : :- CometNativeScan parquet spark_catalog.default.store_sales (123) + : +- ReusedExchange (124) + :- CometProject (133) + : +- CometBroadcastHashJoin (132) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (127) + : +- CometBroadcastExchange (131) + : +- CometProject (130) + : +- CometFilter (129) + : +- CometNativeScan parquet spark_catalog.default.date_dim (128) + +- CometProject (137) + +- CometBroadcastHashJoin (136) + :- CometNativeScan parquet spark_catalog.default.web_sales (134) + +- ReusedExchange (135) + + +(123) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#163)] +ReadSchema: struct + +(124) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#164] + +(125) CometBroadcastHashJoin +Left output [3]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163] +Right output [1]: [d_date_sk#164] +Arguments: [ss_sold_date_sk#163], [d_date_sk#164], Inner, BuildRight + +(126) CometProject +Input [4]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163, d_date_sk#164] +Arguments: [quantity#165, list_price#166], [ss_quantity#161 AS quantity#165, ss_list_price#162 AS list_price#166] + +(127) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#169)] +ReadSchema: struct + +(128) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#170, d_year#171] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(129) CometFilter +Input [2]: [d_date_sk#170, d_year#171] +Condition : (((isnotnull(d_year#171) AND (d_year#171 >= 1998)) AND (d_year#171 <= 2000)) AND isnotnull(d_date_sk#170)) + +(130) CometProject +Input [2]: [d_date_sk#170, d_year#171] +Arguments: [d_date_sk#170], [d_date_sk#170] + +(131) CometBroadcastExchange +Input [1]: [d_date_sk#170] +Arguments: [d_date_sk#170] + +(132) CometBroadcastHashJoin +Left output [3]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169] +Right output [1]: [d_date_sk#170] +Arguments: [cs_sold_date_sk#169], [d_date_sk#170], Inner, BuildRight + +(133) CometProject +Input [4]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169, d_date_sk#170] +Arguments: [quantity#172, list_price#173], [cs_quantity#167 AS quantity#172, cs_list_price#168 AS list_price#173] + +(134) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#176)] +ReadSchema: struct + +(135) ReusedExchange [Reuses operator id: 131] +Output [1]: [d_date_sk#177] + +(136) CometBroadcastHashJoin +Left output [3]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176] +Right output [1]: [d_date_sk#177] +Arguments: [ws_sold_date_sk#176], [d_date_sk#177], Inner, BuildRight + +(137) CometProject +Input [4]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176, d_date_sk#177] +Arguments: [quantity#178, list_price#179], [ws_quantity#174 AS quantity#178, ws_list_price#175 AS list_price#179] + +(138) CometUnion +Child 0 Input [2]: [quantity#165, list_price#166] +Child 1 Input [2]: [quantity#172, list_price#173] +Child 2 Input [2]: [quantity#178, list_price#179] + +(139) CometHashAggregate +Input [2]: [quantity#165, list_price#166] +Keys: [] +Functions [1]: [partial_avg((cast(quantity#165 as decimal(10,0)) * list_price#166))] + +(140) CometExchange +Input [2]: [sum#180, count#181] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(141) CometHashAggregate +Input [2]: [sum#180, count#181] +Keys: [] +Functions [1]: [avg((cast(quantity#165 as decimal(10,0)) * list_price#166))] + +(142) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#182] + +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] + +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/extended.txt new file mode 100644 index 0000000000..c24fc36bd9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/extended.txt @@ -0,0 +1,2030 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 2007 out of 2022 eligible operators (99%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..ff1ccebc20 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_datafusion/simplified.txt @@ -0,0 +1,149 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #14 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #10 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #15 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #3 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #4 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #6 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #13 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id] #18 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] + CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id] #19 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] + CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel] #20 + CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] + CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange #21 + CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/explain.txt new file mode 100644 index 0000000000..2945222916 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/explain.txt @@ -0,0 +1,805 @@ +== Physical Plan == +* CometColumnarToRow (146) ++- CometTakeOrderedAndProject (145) + +- CometUnion (144) + :- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometFilter (16) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.item (32) + :- CometHashAggregate (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (51) + : : : : +- CometBroadcastHashJoin (50) + : : : : :- CometProject (48) + : : : : : +- CometBroadcastHashJoin (47) + : : : : : :- CometProject (45) + : : : : : : +- CometBroadcastHashJoin (44) + : : : : : : :- CometFilter (42) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (41) + : : : : : : +- ReusedExchange (43) + : : : : : +- ReusedExchange (46) + : : : : +- ReusedExchange (49) + : : : +- CometBroadcastExchange (55) + : : : +- CometProject (54) + : : : +- CometFilter (53) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (52) + : : +- ReusedExchange (58) + : +- ReusedExchange (61) + :- CometHashAggregate (92) + : +- CometExchange (91) + : +- CometHashAggregate (90) + : +- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometProject (86) + : : +- CometBroadcastHashJoin (85) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (77) + : : : : +- CometBroadcastHashJoin (76) + : : : : :- CometProject (74) + : : : : : +- CometBroadcastHashJoin (73) + : : : : : :- CometProject (71) + : : : : : : +- CometBroadcastHashJoin (70) + : : : : : : :- CometFilter (68) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (67) + : : : : : : +- ReusedExchange (69) + : : : : : +- ReusedExchange (72) + : : : : +- ReusedExchange (75) + : : : +- CometBroadcastExchange (81) + : : : +- CometProject (80) + : : : +- CometFilter (79) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (78) + : : +- ReusedExchange (84) + : +- ReusedExchange (87) + :- CometHashAggregate (118) + : +- CometExchange (117) + : +- CometHashAggregate (116) + : +- CometProject (115) + : +- CometBroadcastHashJoin (114) + : :- CometProject (112) + : : +- CometBroadcastHashJoin (111) + : : :- CometProject (109) + : : : +- CometBroadcastHashJoin (108) + : : : :- CometProject (103) + : : : : +- CometBroadcastHashJoin (102) + : : : : :- CometProject (100) + : : : : : +- CometBroadcastHashJoin (99) + : : : : : :- CometProject (97) + : : : : : : +- CometBroadcastHashJoin (96) + : : : : : : :- CometFilter (94) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (93) + : : : : : : +- ReusedExchange (95) + : : : : : +- ReusedExchange (98) + : : : : +- ReusedExchange (101) + : : : +- CometBroadcastExchange (107) + : : : +- CometProject (106) + : : : +- CometFilter (105) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (104) + : : +- ReusedExchange (110) + : +- ReusedExchange (113) + +- CometHashAggregate (143) + +- CometExchange (142) + +- CometHashAggregate (141) + +- CometProject (140) + +- CometBroadcastHashJoin (139) + :- CometProject (135) + : +- CometBroadcastHashJoin (134) + : :- CometProject (132) + : : +- CometBroadcastHashJoin (131) + : : :- CometProject (129) + : : : +- CometBroadcastHashJoin (128) + : : : :- CometProject (126) + : : : : +- CometBroadcastHashJoin (125) + : : : : :- CometProject (123) + : : : : : +- CometBroadcastHashJoin (122) + : : : : : :- CometFilter (120) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (119) + : : : : : +- ReusedExchange (121) + : : : : +- ReusedExchange (124) + : : : +- ReusedExchange (127) + : : +- ReusedExchange (130) + : +- ReusedExchange (133) + +- CometBroadcastExchange (138) + +- CometFilter (137) + +- CometNativeScan parquet spark_catalog.default.item (136) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) + +(3) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#10)) + +(5) CometProject +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] + +(6) CometBroadcastExchange +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] + +(7) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight + +(8) CometProject +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] + +(9) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) + +(11) CometProject +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(12) CometBroadcastExchange +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(13) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight + +(14) CometProject +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(15) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) + +(17) CometBroadcastExchange +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] + +(18) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight + +(19) CometProject +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] + +(20) CometNativeScan parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(21) CometFilter +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) + +(22) CometProject +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] + +(24) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight + +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(26) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) + +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight + +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] + +(38) CometHashAggregate +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(39) CometExchange +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(41) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(42) CometFilter +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) + +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] + +(44) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight + +(45) CometProject +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] + +(46) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] + +(47) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight + +(48) CometProject +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] + +(49) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#70] + +(50) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight + +(51) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] + +(52) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(53) CometFilter +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#72, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) + +(54) CometProject +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#74, ca_country#73], [ca_address_sk#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#72, 2)) AS ca_state#74, ca_country#73] + +(55) CometBroadcastExchange +Input [3]: [ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#74, ca_country#73] + +(56) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight + +(57) CometProject +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] + +(58) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#75] + +(59) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] +Right output [1]: [d_date_sk#75] +Arguments: [cs_sold_date_sk#63], [d_date_sk#75], Inner, BuildRight + +(60) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73, d_date_sk#75] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] + +(61) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#76, i_item_id#77] + +(62) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] +Right output [2]: [i_item_sk#76, i_item_id#77] +Arguments: [cs_item_sk#57], [i_item_sk#76], Inner, BuildRight + +(63) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73, i_item_sk#76, i_item_id#77] +Arguments: [i_item_id#77, ca_country#73, ca_state#74, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84], [i_item_id#77, ca_country#73, ca_state#74, cast(cs_quantity#58 as decimal(12,2)) AS agg1#78, cast(cs_list_price#59 as decimal(12,2)) AS agg2#79, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#80, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#81, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#82, cast(c_birth_year#69 as decimal(12,2)) AS agg6#83, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#84] + +(64) CometHashAggregate +Input [10]: [i_item_id#77, ca_country#73, ca_state#74, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84] +Keys [3]: [i_item_id#77, ca_country#73, ca_state#74] +Functions [7]: [partial_avg(agg1#78), partial_avg(agg2#79), partial_avg(agg3#80), partial_avg(agg4#81), partial_avg(agg5#82), partial_avg(agg6#83), partial_avg(agg7#84)] + +(65) CometExchange +Input [17]: [i_item_id#77, ca_country#73, ca_state#74, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] +Arguments: hashpartitioning(i_item_id#77, ca_country#73, ca_state#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(66) CometHashAggregate +Input [17]: [i_item_id#77, ca_country#73, ca_state#74, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] +Keys [3]: [i_item_id#77, ca_country#73, ca_state#74] +Functions [7]: [avg(agg1#78), avg(agg2#79), avg(agg3#80), avg(agg4#81), avg(agg5#82), avg(agg6#83), avg(agg7#84)] + +(67) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#107)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(68) CometFilter +Input [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Condition : ((isnotnull(cs_bill_cdemo_sk#100) AND isnotnull(cs_bill_customer_sk#99)) AND isnotnull(cs_item_sk#101)) + +(69) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#108, cd_dep_count#109] + +(70) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Right output [2]: [cd_demo_sk#108, cd_dep_count#109] +Arguments: [cs_bill_cdemo_sk#100], [cd_demo_sk#108], Inner, BuildRight + +(71) CometProject +Input [11]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_demo_sk#108, cd_dep_count#109] +Arguments: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109], [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109] + +(72) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] + +(73) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109] +Right output [4]: [c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Arguments: [cs_bill_customer_sk#99], [c_customer_sk#110], Inner, BuildRight + +(74) CometProject +Input [13]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] + +(75) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#114] + +(76) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Right output [1]: [cd_demo_sk#114] +Arguments: [c_current_cdemo_sk#111], [cd_demo_sk#114], Inner, BuildRight + +(77) CometProject +Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113, cd_demo_sk#114] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113] + +(78) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(79) CometFilter +Input [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#116, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#115)) + +(80) CometProject +Input [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Arguments: [ca_address_sk#115, ca_country#117], [ca_address_sk#115, ca_country#117] + +(81) CometBroadcastExchange +Input [2]: [ca_address_sk#115, ca_country#117] +Arguments: [ca_address_sk#115, ca_country#117] + +(82) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113] +Right output [2]: [ca_address_sk#115, ca_country#117] +Arguments: [c_current_addr_sk#112], [ca_address_sk#115], Inner, BuildRight + +(83) CometProject +Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113, ca_address_sk#115, ca_country#117] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117] + +(84) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#118] + +(85) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117] +Right output [1]: [d_date_sk#118] +Arguments: [cs_sold_date_sk#107], [d_date_sk#118], Inner, BuildRight + +(86) CometProject +Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117, d_date_sk#118] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117] + +(87) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#119, i_item_id#120] + +(88) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117] +Right output [2]: [i_item_sk#119, i_item_id#120] +Arguments: [cs_item_sk#101], [i_item_sk#119], Inner, BuildRight + +(89) CometProject +Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117, i_item_sk#119, i_item_id#120] +Arguments: [i_item_id#120, ca_country#117, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127], [i_item_id#120, ca_country#117, cast(cs_quantity#102 as decimal(12,2)) AS agg1#121, cast(cs_list_price#103 as decimal(12,2)) AS agg2#122, cast(cs_coupon_amt#105 as decimal(12,2)) AS agg3#123, cast(cs_sales_price#104 as decimal(12,2)) AS agg4#124, cast(cs_net_profit#106 as decimal(12,2)) AS agg5#125, cast(c_birth_year#113 as decimal(12,2)) AS agg6#126, cast(cd_dep_count#109 as decimal(12,2)) AS agg7#127] + +(90) CometHashAggregate +Input [9]: [i_item_id#120, ca_country#117, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127] +Keys [2]: [i_item_id#120, ca_country#117] +Functions [7]: [partial_avg(agg1#121), partial_avg(agg2#122), partial_avg(agg3#123), partial_avg(agg4#124), partial_avg(agg5#125), partial_avg(agg6#126), partial_avg(agg7#127)] + +(91) CometExchange +Input [16]: [i_item_id#120, ca_country#117, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] +Arguments: hashpartitioning(i_item_id#120, ca_country#117, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(92) CometHashAggregate +Input [16]: [i_item_id#120, ca_country#117, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] +Keys [2]: [i_item_id#120, ca_country#117] +Functions [7]: [avg(agg1#121), avg(agg2#122), avg(agg3#123), avg(agg4#124), avg(agg5#125), avg(agg6#126), avg(agg7#127)] + +(93) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#150)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(94) CometFilter +Input [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Condition : ((isnotnull(cs_bill_cdemo_sk#143) AND isnotnull(cs_bill_customer_sk#142)) AND isnotnull(cs_item_sk#144)) + +(95) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#151, cd_dep_count#152] + +(96) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Right output [2]: [cd_demo_sk#151, cd_dep_count#152] +Arguments: [cs_bill_cdemo_sk#143], [cd_demo_sk#151], Inner, BuildRight + +(97) CometProject +Input [11]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_demo_sk#151, cd_dep_count#152] +Arguments: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152], [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152] + +(98) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] + +(99) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152] +Right output [4]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Arguments: [cs_bill_customer_sk#142], [c_customer_sk#153], Inner, BuildRight + +(100) CometProject +Input [13]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] + +(101) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#157] + +(102) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Right output [1]: [cd_demo_sk#157] +Arguments: [c_current_cdemo_sk#154], [cd_demo_sk#157], Inner, BuildRight + +(103) CometProject +Input [12]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156, cd_demo_sk#157] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156] + +(104) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#158, ca_state#159] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(105) CometFilter +Input [2]: [ca_address_sk#158, ca_state#159] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#159, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#158)) + +(106) CometProject +Input [2]: [ca_address_sk#158, ca_state#159] +Arguments: [ca_address_sk#158], [ca_address_sk#158] + +(107) CometBroadcastExchange +Input [1]: [ca_address_sk#158] +Arguments: [ca_address_sk#158] + +(108) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156] +Right output [1]: [ca_address_sk#158] +Arguments: [c_current_addr_sk#155], [ca_address_sk#158], Inner, BuildRight + +(109) CometProject +Input [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156, ca_address_sk#158] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156] + +(110) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#160] + +(111) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156] +Right output [1]: [d_date_sk#160] +Arguments: [cs_sold_date_sk#150], [d_date_sk#160], Inner, BuildRight + +(112) CometProject +Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156, d_date_sk#160] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156] + +(113) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#161, i_item_id#162] + +(114) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156] +Right output [2]: [i_item_sk#161, i_item_id#162] +Arguments: [cs_item_sk#144], [i_item_sk#161], Inner, BuildRight + +(115) CometProject +Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156, i_item_sk#161, i_item_id#162] +Arguments: [i_item_id#162, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169], [i_item_id#162, cast(cs_quantity#145 as decimal(12,2)) AS agg1#163, cast(cs_list_price#146 as decimal(12,2)) AS agg2#164, cast(cs_coupon_amt#148 as decimal(12,2)) AS agg3#165, cast(cs_sales_price#147 as decimal(12,2)) AS agg4#166, cast(cs_net_profit#149 as decimal(12,2)) AS agg5#167, cast(c_birth_year#156 as decimal(12,2)) AS agg6#168, cast(cd_dep_count#152 as decimal(12,2)) AS agg7#169] + +(116) CometHashAggregate +Input [8]: [i_item_id#162, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169] +Keys [1]: [i_item_id#162] +Functions [7]: [partial_avg(agg1#163), partial_avg(agg2#164), partial_avg(agg3#165), partial_avg(agg4#166), partial_avg(agg5#167), partial_avg(agg6#168), partial_avg(agg7#169)] + +(117) CometExchange +Input [15]: [i_item_id#162, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] +Arguments: hashpartitioning(i_item_id#162, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(118) CometHashAggregate +Input [15]: [i_item_id#162, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] +Keys [1]: [i_item_id#162] +Functions [7]: [avg(agg1#163), avg(agg2#164), avg(agg3#165), avg(agg4#166), avg(agg5#167), avg(agg6#168), avg(agg7#169)] + +(119) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#192)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(120) CometFilter +Input [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Condition : ((isnotnull(cs_bill_cdemo_sk#185) AND isnotnull(cs_bill_customer_sk#184)) AND isnotnull(cs_item_sk#186)) + +(121) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#193, cd_dep_count#194] + +(122) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Right output [2]: [cd_demo_sk#193, cd_dep_count#194] +Arguments: [cs_bill_cdemo_sk#185], [cd_demo_sk#193], Inner, BuildRight + +(123) CometProject +Input [11]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_demo_sk#193, cd_dep_count#194] +Arguments: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194], [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194] + +(124) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] + +(125) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194] +Right output [4]: [c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Arguments: [cs_bill_customer_sk#184], [c_customer_sk#195], Inner, BuildRight + +(126) CometProject +Input [13]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] + +(127) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#199] + +(128) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Right output [1]: [cd_demo_sk#199] +Arguments: [c_current_cdemo_sk#196], [cd_demo_sk#199], Inner, BuildRight + +(129) CometProject +Input [12]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198, cd_demo_sk#199] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198] + +(130) ReusedExchange [Reuses operator id: 107] +Output [1]: [ca_address_sk#200] + +(131) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198] +Right output [1]: [ca_address_sk#200] +Arguments: [c_current_addr_sk#197], [ca_address_sk#200], Inner, BuildRight + +(132) CometProject +Input [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198, ca_address_sk#200] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198] + +(133) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#201] + +(134) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198] +Right output [1]: [d_date_sk#201] +Arguments: [cs_sold_date_sk#192], [d_date_sk#201], Inner, BuildRight + +(135) CometProject +Input [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198, d_date_sk#201] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198] + +(136) CometNativeScan parquet spark_catalog.default.item +Output [1]: [i_item_sk#202] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(137) CometFilter +Input [1]: [i_item_sk#202] +Condition : isnotnull(i_item_sk#202) + +(138) CometBroadcastExchange +Input [1]: [i_item_sk#202] +Arguments: [i_item_sk#202] + +(139) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198] +Right output [1]: [i_item_sk#202] +Arguments: [cs_item_sk#186], [i_item_sk#202], Inner, BuildRight + +(140) CometProject +Input [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198, i_item_sk#202] +Arguments: [agg1#203, agg2#204, agg3#205, agg4#206, agg5#207, agg6#208, agg7#209], [cast(cs_quantity#187 as decimal(12,2)) AS agg1#203, cast(cs_list_price#188 as decimal(12,2)) AS agg2#204, cast(cs_coupon_amt#190 as decimal(12,2)) AS agg3#205, cast(cs_sales_price#189 as decimal(12,2)) AS agg4#206, cast(cs_net_profit#191 as decimal(12,2)) AS agg5#207, cast(c_birth_year#198 as decimal(12,2)) AS agg6#208, cast(cd_dep_count#194 as decimal(12,2)) AS agg7#209] + +(141) CometHashAggregate +Input [7]: [agg1#203, agg2#204, agg3#205, agg4#206, agg5#207, agg6#208, agg7#209] +Keys: [] +Functions [7]: [partial_avg(agg1#203), partial_avg(agg2#204), partial_avg(agg3#205), partial_avg(agg4#206), partial_avg(agg5#207), partial_avg(agg6#208), partial_avg(agg7#209)] + +(142) CometExchange +Input [14]: [sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221, sum#222, count#223] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(143) CometHashAggregate +Input [14]: [sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221, sum#222, count#223] +Keys: [] +Functions [7]: [avg(agg1#203), avg(agg2#204), avg(agg3#205), avg(agg4#206), avg(agg5#207), avg(agg6#208), avg(agg7#209)] + +(144) CometUnion +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] +Child 1 Input [11]: [i_item_id#77, ca_country#73, ca_state#74, county#231, agg1#232, agg2#233, agg3#234, agg4#235, agg5#236, agg6#237, agg7#238] +Child 2 Input [11]: [i_item_id#120, ca_country#117, ca_state#239, county#240, agg1#241, agg2#242, agg3#243, agg4#244, agg5#245, agg6#246, agg7#247] +Child 3 Input [11]: [i_item_id#162, ca_country#248, ca_state#249, county#250, agg1#251, agg2#252, agg3#253, agg4#254, agg5#255, agg6#256, agg7#257] +Child 4 Input [11]: [i_item_id#258, ca_country#259, ca_state#260, county#261, agg1#262, agg2#263, agg3#264, agg4#265, agg5#266, agg6#267, agg7#268] + +(145) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#224,agg2#225,agg3#226,agg4#227,agg5#228,agg6#229,agg7#230]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] + +(146) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/extended.txt new file mode 100644 index 0000000000..59e46c3c8e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/extended.txt @@ -0,0 +1,204 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 201 out of 201 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..a5655241a4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_datafusion/simplified.txt @@ -0,0 +1,148 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state] #8 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 + CometProject [ca_state] [ca_address_sk,ca_state,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country] #10 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_country] #11 + CometProject [ca_address_sk,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id] #12 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk] #13 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange #14 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + ReusedExchange [ca_address_sk] #13 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [i_item_sk] #15 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/explain.txt new file mode 100644 index 0000000000..489aebe7c2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/explain.txt @@ -0,0 +1,137 @@ +== Physical Plan == +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Condition : isnotnull(cs_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS _w0#18] + +(19) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] + +(22) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] + +(24) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/extended.txt new file mode 100644 index 0000000000..1c660ade5e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/extended.txt @@ -0,0 +1,26 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#1, i_item_desc#2, i_category#3, i_class#4, i_current_price#5, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#6))#7,17,2) AS itemrevenue#8, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#6))#7,17,2) AS _w0#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 18 out of 22 eligible operators (81%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/simplified.txt new file mode 100644 index 0000000000..289715c3a2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_datafusion/simplified.txt @@ -0,0 +1,30 @@ +TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] + WholeStageCodegen (3) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/explain.txt new file mode 100644 index 0000000000..6e5dfb8917 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/explain.txt @@ -0,0 +1,150 @@ +== Physical Plan == +TakeOrderedAndProject (26) ++- * HashAggregate (25) + +- * CometColumnarToRow (24) + +- CometColumnarExchange (23) + +- * HashAggregate (22) + +- * Expand (21) + +- * Project (20) + +- * BroadcastNestedLoopJoin Inner BuildRight (19) + :- * CometColumnarToRow (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- BroadcastExchange (18) + +- * CometColumnarToRow (17) + +- CometNativeScan parquet spark_catalog.default.warehouse (16) + + +(1) CometNativeScan parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#3)] +PushedFilters: [IsNotNull(inv_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Condition : isnotnull(inv_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] + +(9) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Condition : isnotnull(i_item_sk#6) + +(11) CometProject +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#7, 50)) AS i_brand#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#10, 50)) AS i_product_name#14] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(13) CometBroadcastHashJoin +Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(14) CometProject +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(15) CometColumnarToRow [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(16) CometNativeScan parquet spark_catalog.default.warehouse +Output: [] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +ReadSchema: struct<> + +(17) CometColumnarToRow [codegen id : 1] +Input: [] + +(18) BroadcastExchange +Input: [] +Arguments: IdentityBroadcastMode, [plan_id=1] + +(19) BroadcastNestedLoopJoin [codegen id : 2] +Join type: Inner +Join condition: None + +(20) Project [codegen id : 2] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(21) Expand [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] + +(22) HashAggregate [codegen id : 2] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Functions [1]: [partial_avg(inv_quantity_on_hand#2)] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(23) CometColumnarExchange +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometColumnarToRow [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(25) HashAggregate [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Functions [1]: [avg(inv_quantity_on_hand#2)] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] + +(26) TakeOrderedAndProject +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/extended.txt new file mode 100644 index 0000000000..e4737a8934 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/extended.txt @@ -0,0 +1,28 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Project + +- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 16 out of 23 eligible operators (69%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/simplified.txt new file mode 100644 index 0000000000..199ad41e45 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_datafusion/simplified.txt @@ -0,0 +1,33 @@ +TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] + WholeStageCodegen (3) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometNativeScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/explain.txt new file mode 100644 index 0000000000..baefe2c616 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/explain.txt @@ -0,0 +1,268 @@ +== Physical Plan == +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometUnion (45) + :- CometHashAggregate (24) + : +- CometHashAggregate (23) + : +- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.warehouse (15) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometHashAggregate (26) + : +- ReusedExchange (25) + :- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + :- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometHashAggregate (36) + : +- ReusedExchange (35) + +- CometHashAggregate (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometHashAggregate (41) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] +Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] + +(9) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) + +(11) CometProject +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#8, 50)) AS i_brand#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#11, 50)) AS i_product_name#15] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(13) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight + +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(15) CometNativeScan parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) + +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] + +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight + +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(20) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [partial_avg(inv_quantity_on_hand#3)] + +(21) CometExchange +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#3)] + +(23) CometHashAggregate +Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, qoh#19] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [partial_avg(qoh#19)] + +(24) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#20, count#21] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(qoh#19)] + +(25) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#22, i_brand#23, i_class#24, i_category#25, sum#26, count#27] + +(26) CometHashAggregate +Input [6]: [i_product_name#22, i_brand#23, i_class#24, i_category#25, sum#26, count#27] +Keys [4]: [i_product_name#22, i_brand#23, i_class#24, i_category#25] +Functions [1]: [avg(inv_quantity_on_hand#28)] + +(27) CometHashAggregate +Input [4]: [i_product_name#22, i_brand#23, i_class#24, qoh#29] +Keys [3]: [i_product_name#22, i_brand#23, i_class#24] +Functions [1]: [partial_avg(qoh#29)] + +(28) CometExchange +Input [5]: [i_product_name#22, i_brand#23, i_class#24, sum#30, count#31] +Arguments: hashpartitioning(i_product_name#22, i_brand#23, i_class#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometHashAggregate +Input [5]: [i_product_name#22, i_brand#23, i_class#24, sum#30, count#31] +Keys [3]: [i_product_name#22, i_brand#23, i_class#24] +Functions [1]: [avg(qoh#29)] + +(30) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#32, i_brand#33, i_class#34, i_category#35, sum#36, count#37] + +(31) CometHashAggregate +Input [6]: [i_product_name#32, i_brand#33, i_class#34, i_category#35, sum#36, count#37] +Keys [4]: [i_product_name#32, i_brand#33, i_class#34, i_category#35] +Functions [1]: [avg(inv_quantity_on_hand#38)] + +(32) CometHashAggregate +Input [3]: [i_product_name#32, i_brand#33, qoh#39] +Keys [2]: [i_product_name#32, i_brand#33] +Functions [1]: [partial_avg(qoh#39)] + +(33) CometExchange +Input [4]: [i_product_name#32, i_brand#33, sum#40, count#41] +Arguments: hashpartitioning(i_product_name#32, i_brand#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [4]: [i_product_name#32, i_brand#33, sum#40, count#41] +Keys [2]: [i_product_name#32, i_brand#33] +Functions [1]: [avg(qoh#39)] + +(35) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#42, i_brand#43, i_class#44, i_category#45, sum#46, count#47] + +(36) CometHashAggregate +Input [6]: [i_product_name#42, i_brand#43, i_class#44, i_category#45, sum#46, count#47] +Keys [4]: [i_product_name#42, i_brand#43, i_class#44, i_category#45] +Functions [1]: [avg(inv_quantity_on_hand#48)] + +(37) CometHashAggregate +Input [2]: [i_product_name#42, qoh#49] +Keys [1]: [i_product_name#42] +Functions [1]: [partial_avg(qoh#49)] + +(38) CometExchange +Input [3]: [i_product_name#42, sum#50, count#51] +Arguments: hashpartitioning(i_product_name#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometHashAggregate +Input [3]: [i_product_name#42, sum#50, count#51] +Keys [1]: [i_product_name#42] +Functions [1]: [avg(qoh#49)] + +(40) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#52, i_brand#53, i_class#54, i_category#55, sum#56, count#57] + +(41) CometHashAggregate +Input [6]: [i_product_name#52, i_brand#53, i_class#54, i_category#55, sum#56, count#57] +Keys [4]: [i_product_name#52, i_brand#53, i_class#54, i_category#55] +Functions [1]: [avg(inv_quantity_on_hand#58)] + +(42) CometHashAggregate +Input [1]: [qoh#59] +Keys: [] +Functions [1]: [partial_avg(qoh#59)] + +(43) CometExchange +Input [2]: [sum#60, count#61] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(44) CometHashAggregate +Input [2]: [sum#60, count#61] +Keys: [] +Functions [1]: [avg(qoh#59)] + +(45) CometUnion +Child 0 Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] +Child 1 Input [5]: [i_product_name#22, i_brand#23, i_class#24, i_category#67, qoh#68] +Child 2 Input [5]: [i_product_name#32, i_brand#33, i_class#69, i_category#70, qoh#71] +Child 3 Input [5]: [i_product_name#42, i_brand#72, i_class#73, i_category#74, qoh#75] +Child 4 Input [5]: [i_product_name#76, i_brand#77, i_class#78, i_category#79, qoh#80] + +(46) CometTakeOrderedAndProject +Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#66 ASC NULLS FIRST,i_product_name#62 ASC NULLS FIRST,i_brand#63 ASC NULLS FIRST,i_class#64 ASC NULLS FIRST,i_category#65 ASC NULLS FIRST], output=[i_product_name#62,i_brand#63,i_class#64,i_category#65,qoh#66]), [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66], 100, 0, [qoh#66 ASC NULLS FIRST, i_product_name#62 ASC NULLS FIRST, i_brand#63 ASC NULLS FIRST, i_class#64 ASC NULLS FIRST, i_category#65 ASC NULLS FIRST], [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] + +(47) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/extended.txt new file mode 100644 index 0000000000..ed898ac4d5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/extended.txt @@ -0,0 +1,129 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 126 out of 126 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..c00ab4515c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_datafusion/simplified.txt @@ -0,0 +1,49 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometUnion [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,i_product_name,i_brand,i_class,i_category,avg(qoh)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,i_category,sum,count] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,sum,count] + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand,i_class] #5 + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] + CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand] #6 + CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] + CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name] #7 + CometHashAggregate [qoh] [i_product_name,sum,count] + CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange #8 + CometHashAggregate [qoh] [sum,count] + CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/explain.txt new file mode 100644 index 0000000000..6ca8ff436b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/explain.txt @@ -0,0 +1,453 @@ +== Physical Plan == +* CometColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- * Filter (47) + +- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) + +(3) CometProject +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] + +(4) CometExchange +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(5) CometSort +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] + +(6) CometNativeScan parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(7) CometFilter +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) + +(8) CometProject +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] + +(9) CometExchange +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(10) CometSort +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] + +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner + +(12) CometProject +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] + +(13) CometNativeScan parquet spark_catalog.default.store +Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)))) + +(15) CometProject +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)) AS s_zip#16] + +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] + +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight + +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] + +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] +ReadSchema: struct + +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((isnotnull(i_color#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) = pale )) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#19, 20)) AS i_size#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) AS i_color#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#21, 10)) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Condition : ((isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) AND isnotnull(c_birth_country#30)) + +(27) CometProject +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [c_customer_sk#26, c_current_addr_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#32, c_birth_country#30] + +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(31) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Condition : ((isnotnull(ca_address_sk#33) AND isnotnull(ca_country#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#35, 10)))) + +(34) CometProject +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Arguments: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36], [ca_address_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#34, 2)) AS ca_state#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#35, 10)) AS ca_zip#38, ca_country#36] + +(35) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] + +(36) BroadcastExchange +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] +Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[3, string, true]), input[2, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#27, c_birth_country#30, s_zip#16] +Right keys [3]: [ca_address_sk#33, upper(ca_country#36), ca_zip#38] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30, ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] + +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum#39] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] + +(40) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#41] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#41,17,2) AS netpaid#42] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, netpaid#42] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [partial_sum(netpaid#42)] +Aggregate Attributes [2]: [sum#43, isEmpty#44] +Results [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(44) CometColumnarExchange +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [sum(netpaid#42)] +Aggregate Attributes [1]: [sum(netpaid#42)#47] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, sum(netpaid#42)#47 AS paid#48] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Condition : (isnotnull(paid#48) AND (cast(paid#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8)))) + +(48) CometColumnarExchange +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: rangepartitioning(c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(49) CometSort +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48], [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] + +(50) CometColumnarToRow [codegen id : 5] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* HashAggregate (80) ++- * CometColumnarToRow (79) + +- CometColumnarExchange (78) + +- * HashAggregate (77) + +- * HashAggregate (76) + +- * CometColumnarToRow (75) + +- CometColumnarExchange (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- * CometColumnarToRow (69) + : +- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (56) + : : : : +- CometSortMergeJoin (55) + : : : : :- CometSort (52) + : : : : : +- ReusedExchange (51) + : : : : +- CometSort (54) + : : : : +- ReusedExchange (53) + : : : +- ReusedExchange (57) + : : +- CometBroadcastExchange (63) + : : +- CometProject (62) + : : +- CometFilter (61) + : : +- CometNativeScan parquet spark_catalog.default.item (60) + : +- ReusedExchange (66) + +- ReusedExchange (70) + + +(51) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] + +(52) CometSort +Input [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55], [ss_ticket_number#54 ASC NULLS FIRST, ss_item_sk#51 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#56, sr_ticket_number#57] + +(54) CometSort +Input [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [sr_item_sk#56, sr_ticket_number#57], [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Right output [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_ticket_number#54, ss_item_sk#51], [sr_ticket_number#57, sr_item_sk#56], Inner + +(56) CometProject +Input [7]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55, sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55], [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] + +(57) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#58, s_store_name#59, s_state#60, s_zip#61] + +(58) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] +Right output [4]: [s_store_sk#58, s_store_name#59, s_state#60, s_zip#61] +Arguments: [ss_store_sk#53], [s_store_sk#58], Inner, BuildRight + +(59) CometProject +Input [8]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55, s_store_sk#58, s_store_name#59, s_state#60, s_zip#61] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61], [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61] + +(60) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#62, i_current_price#63, i_size#64, i_color#65, i_units#66, i_manager_id#67] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(61) CometFilter +Input [6]: [i_item_sk#62, i_current_price#63, i_size#64, i_color#65, i_units#66, i_manager_id#67] +Condition : isnotnull(i_item_sk#62) + +(62) CometProject +Input [6]: [i_item_sk#62, i_current_price#63, i_size#64, i_color#65, i_units#66, i_manager_id#67] +Arguments: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67], [i_item_sk#62, i_current_price#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#64, 20)) AS i_size#68, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#65, 20)) AS i_color#69, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#66, 10)) AS i_units#70, i_manager_id#67] + +(63) CometBroadcastExchange +Input [6]: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Arguments: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] + +(64) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61] +Right output [6]: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Arguments: [ss_item_sk#51], [i_item_sk#62], Inner, BuildRight + +(65) CometProject +Input [12]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Arguments: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67], [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] + +(66) ReusedExchange [Reuses operator id: 28] +Output [5]: [c_customer_sk#71, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] + +(67) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Right output [5]: [c_customer_sk#71, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] +Arguments: [ss_customer_sk#52], [c_customer_sk#71], Inner, BuildRight + +(68) CometProject +Input [15]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_customer_sk#71, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] +Arguments: [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75], [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] + +(69) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] + +(70) ReusedExchange [Reuses operator id: 36] +Output [4]: [ca_address_sk#76, ca_state#77, ca_zip#78, ca_country#79] + +(71) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#72, c_birth_country#75, s_zip#61] +Right keys [3]: [ca_address_sk#76, upper(ca_country#79), ca_zip#78] +Join type: Inner +Join condition: None + +(72) Project [codegen id : 2] +Output [11]: [ss_net_paid#55, s_store_name#59, s_state#60, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_first_name#73, c_last_name#74, ca_state#77] +Input [17]: [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75, ca_address_sk#76, ca_state#77, ca_zip#78, ca_country#79] + +(73) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#55, s_store_name#59, s_state#60, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_first_name#73, c_last_name#74, ca_state#77] +Keys [10]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum#80] +Results [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] + +(74) CometColumnarExchange +Input [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] +Arguments: hashpartitioning(c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(75) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] + +(76) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] +Keys [10]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68] +Functions [1]: [sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#55))#41] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#55))#41,17,2) AS netpaid#82] + +(77) HashAggregate [codegen id : 3] +Input [1]: [netpaid#82] +Keys: [] +Functions [1]: [partial_avg(netpaid#82)] +Aggregate Attributes [2]: [sum#83, count#84] +Results [2]: [sum#85, count#86] + +(78) CometColumnarExchange +Input [2]: [sum#85, count#86] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(79) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#85, count#86] + +(80) HashAggregate [codegen id : 4] +Input [2]: [sum#85, count#86] +Keys: [] +Functions [1]: [avg(netpaid#82)] +Aggregate Attributes [1]: [avg(netpaid#82)#87] +Results [1]: [(0.05 * avg(netpaid#82)#87) AS (0.05 * avg(netpaid))#88] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/extended.txt new file mode 100644 index 0000000000..13317e59ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/extended.txt @@ -0,0 +1,99 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Filter + : +- Subquery + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 72 out of 88 eligible operators (81%). Final plan contains 9 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/simplified.txt new file mode 100644 index 0000000000..b24e51723a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_datafusion/simplified.txt @@ -0,0 +1,99 @@ +WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [c_last_name,c_first_name,s_store_name,paid] + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (4) + Filter [paid] + Subquery #1 + WholeStageCodegen (4) + HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/explain.txt new file mode 100644 index 0000000000..92797e4ffe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/explain.txt @@ -0,0 +1,400 @@ +== Physical Plan == +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometUnion (69) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.store (15) + : +- CometBroadcastExchange (24) + : +- CometProject (23) + : +- CometFilter (22) + : +- CometNativeScan parquet spark_catalog.default.item (21) + :- CometHashAggregate (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (37) + : : : +- CometBroadcastHashJoin (36) + : : : :- CometProject (34) + : : : : +- CometBroadcastHashJoin (33) + : : : : :- CometFilter (31) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (30) + : : : : +- ReusedExchange (32) + : : : +- ReusedExchange (35) + : : +- CometBroadcastExchange (41) + : : +- CometProject (40) + : : +- CometFilter (39) + : : +- CometNativeScan parquet spark_catalog.default.store (38) + : +- ReusedExchange (44) + +- CometHashAggregate (68) + +- CometExchange (67) + +- CometHashAggregate (66) + +- CometProject (65) + +- CometBroadcastHashJoin (64) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- ReusedExchange (58) + +- CometBroadcastExchange (63) + +- CometFilter (62) + +- CometNativeScan parquet spark_catalog.default.item (61) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) + +(3) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = W)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = Primary )) AND isnotnull(cd_demo_sk#9)) + +(5) CometProject +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] + +(6) CometBroadcastExchange +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] + +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#15, s_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) = TN)) AND isnotnull(s_store_sk#15)) + +(17) CometProject +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) AS s_state#17] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] + +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight + +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] + +(21) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) + +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] + +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] + +(27) CometHashAggregate +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(28) CometExchange +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(30) CometNativeScan parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(31) CometFilter +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) + +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#43] + +(33) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight + +(34) CometProject +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] + +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#44] + +(36) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight + +(37) CometProject +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(38) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_state#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(39) CometFilter +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((isnotnull(s_state#46) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#46, 2)) = TN)) AND isnotnull(s_store_sk#45)) + +(40) CometProject +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] + +(41) CometBroadcastExchange +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] + +(42) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight + +(43) CometProject +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(44) ReusedExchange [Reuses operator id: 24] +Output [2]: [i_item_sk#47, i_item_id#48] + +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#48] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight + +(46) CometProject +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#48] +Arguments: [i_item_id#48, agg1#49, agg2#50, agg3#51, agg4#52], [i_item_id#48, ss_quantity#38 AS agg1#49, ss_list_price#39 AS agg2#50, ss_coupon_amt#41 AS agg3#51, ss_sales_price#40 AS agg4#52] + +(47) CometHashAggregate +Input [5]: [i_item_id#48, agg1#49, agg2#50, agg3#51, agg4#52] +Keys [1]: [i_item_id#48] +Functions [4]: [partial_avg(agg1#49), partial_avg(UnscaledValue(agg2#50)), partial_avg(UnscaledValue(agg3#51)), partial_avg(UnscaledValue(agg4#52))] + +(48) CometExchange +Input [9]: [i_item_id#48, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] +Arguments: hashpartitioning(i_item_id#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(49) CometHashAggregate +Input [9]: [i_item_id#48, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] +Keys [1]: [i_item_id#48] +Functions [4]: [avg(agg1#49), avg(UnscaledValue(agg2#50)), avg(UnscaledValue(agg3#51)), avg(UnscaledValue(agg4#52))] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#68)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Condition : ((isnotnull(ss_cdemo_sk#62) AND isnotnull(ss_store_sk#63)) AND isnotnull(ss_item_sk#61)) + +(52) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#69] + +(53) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Right output [1]: [cd_demo_sk#69] +Arguments: [ss_cdemo_sk#62], [cd_demo_sk#69], Inner, BuildRight + +(54) CometProject +Input [9]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68, cd_demo_sk#69] +Arguments: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68], [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] + +(55) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#70] + +(56) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Right output [1]: [d_date_sk#70] +Arguments: [ss_sold_date_sk#68], [d_date_sk#70], Inner, BuildRight + +(57) CometProject +Input [8]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68, d_date_sk#70] +Arguments: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67], [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] + +(58) ReusedExchange [Reuses operator id: 41] +Output [1]: [s_store_sk#71] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] +Right output [1]: [s_store_sk#71] +Arguments: [ss_store_sk#63], [s_store_sk#71], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, s_store_sk#71] +Arguments: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67], [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] + +(61) CometNativeScan parquet spark_catalog.default.item +Output [1]: [i_item_sk#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(62) CometFilter +Input [1]: [i_item_sk#72] +Condition : isnotnull(i_item_sk#72) + +(63) CometBroadcastExchange +Input [1]: [i_item_sk#72] +Arguments: [i_item_sk#72] + +(64) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] +Right output [1]: [i_item_sk#72] +Arguments: [ss_item_sk#61], [i_item_sk#72], Inner, BuildRight + +(65) CometProject +Input [6]: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, i_item_sk#72] +Arguments: [agg1#73, agg2#74, agg3#75, agg4#76], [ss_quantity#64 AS agg1#73, ss_list_price#65 AS agg2#74, ss_coupon_amt#67 AS agg3#75, ss_sales_price#66 AS agg4#76] + +(66) CometHashAggregate +Input [4]: [agg1#73, agg2#74, agg3#75, agg4#76] +Keys: [] +Functions [4]: [partial_avg(agg1#73), partial_avg(UnscaledValue(agg2#74)), partial_avg(UnscaledValue(agg3#75)), partial_avg(UnscaledValue(agg4#76))] + +(67) CometExchange +Input [8]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(68) CometHashAggregate +Input [8]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84] +Keys: [] +Functions [4]: [avg(agg1#73), avg(UnscaledValue(agg2#74)), avg(UnscaledValue(agg3#75)), avg(UnscaledValue(agg4#76))] + +(69) CometUnion +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] +Child 1 Input [7]: [i_item_id#48, s_state#90, g_state#91, agg1#92, agg2#93, agg3#94, agg4#95] +Child 2 Input [7]: [i_item_id#96, s_state#97, g_state#98, agg1#99, agg2#100, agg3#101, agg4#102] + +(70) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#85,agg1#86,agg2#87,agg3#88,agg4#89]), [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] + +(71) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/extended.txt new file mode 100644 index 0000000000..180236b925 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/extended.txt @@ -0,0 +1,91 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 88 out of 88 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..a7b0657337 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_datafusion/simplified.txt @@ -0,0 +1,73 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id,s_state] #1 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id] #6 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [s_store_sk] #7 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange #8 + CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #7 + CometBroadcastExchange [i_item_sk] #9 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/explain.txt new file mode 100644 index 0000000000..46d2e9bdcb --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/explain.txt @@ -0,0 +1,189 @@ +== Physical Plan == +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] + +(9) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) + +(11) CometProject +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] + +(12) CometBroadcastExchange +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] + +(15) CometNativeScan parquet spark_catalog.default.household_demographics +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) + +(17) CometProject +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] + +(18) CometBroadcastExchange +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] + +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight + +(20) CometProject +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] + +(21) CometHashAggregate +Input [2]: [ss_customer_sk#1, ss_ticket_number#4] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [partial_count(1)] + +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [count(1)] + +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) + +(25) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] + +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight + +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] + +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/simplified.txt new file mode 100644 index 0000000000..cb3f282260 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/simplified.txt @@ -0,0 +1,35 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/explain.txt new file mode 100644 index 0000000000..f1b4f2a73d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/explain.txt @@ -0,0 +1,269 @@ +== Physical Plan == +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) + +(3) CometNativeScan parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct + +(4) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) + +(6) CometProject +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#6] +Arguments: [ss_customer_sk#6] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight + +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct + +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#13] + +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] + +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] + +(18) BroadcastExchange +Input [1]: [ws_bill_customer_sk#11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] + +(19) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [ws_bill_customer_sk#11] +Join type: ExistenceJoin(exists#2) +Join condition: None + +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct + +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [cs_ship_customer_sk#14] +Join type: ExistenceJoin(exists#1) +Join condition: None + +(27) Filter [codegen id : 5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] +Condition : (exists#2 OR exists#1) + +(28) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] + +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) + +(31) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] + +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] + +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#5] +Right keys [1]: [ca_address_sk#17] +Join type: Inner +Join condition: None + +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] + +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) + +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_cdemo_sk#4] +Right keys [1]: [cd_demo_sk#20] +Join type: Inner +Join condition: None + +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#23)#55, max(cd_dep_count#23)#56, sum(cd_dep_count#23)#57, avg(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, sum(cd_dep_employed_count#24)#60, avg(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, sum(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, count(1)#54 AS cnt1#64, avg(cd_dep_count#23)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#23)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#24)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#24)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#25)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#25)#63 AS sum(cd_dep_college_count)#75] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/simplified.txt new file mode 100644 index 0000000000..4b9be3b971 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_datafusion/simplified.txt @@ -0,0 +1,63 @@ +TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + WholeStageCodegen (6) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/explain.txt new file mode 100644 index 0000000000..2eea275459 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/explain.txt @@ -0,0 +1,230 @@ +== Physical Plan == +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) + + +(1) CometNativeScan parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) + +(3) CometNativeScan parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct + +(4) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) + +(6) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#4] +Arguments: [ss_customer_sk#4] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight + +(12) CometNativeScan parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct + +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#11] + +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customsk#12], [ws_bill_customer_sk#9 AS customsk#12] + +(16) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +ReadSchema: struct + +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#15] + +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight + +(19) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customsk#16], [cs_ship_customer_sk#13 AS customsk#16] + +(20) CometUnion +Child 0 Input [1]: [customsk#12] +Child 1 Input [1]: [customsk#16] + +(21) CometBroadcastExchange +Input [1]: [customsk#12] +Arguments: [customsk#12] + +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customsk#12] +Arguments: [c_customer_sk#1], [customsk#12], LeftSemi, BuildRight + +(23) CometProject +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] + +(24) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) + +(26) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] + +(27) CometBroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: [ca_address_sk#17, ca_state#19] + +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [2]: [ca_address_sk#17, ca_state#19] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight + +(29) CometProject +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#19] +Arguments: [c_current_cdemo_sk#2, ca_state#19], [c_current_cdemo_sk#2, ca_state#19] + +(30) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) + +(32) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(33) CometBroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(34) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, ca_state#19] +Right output [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#20], Inner, BuildRight + +(35) CometProject +Input [8]: [c_current_cdemo_sk#2, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(36) CometHashAggregate +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] + +(37) CometExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(38) CometHashAggregate +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] + +(39) CometTakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#19 ASC NULLS FIRST,cd_gender#26 ASC NULLS FIRST,cd_marital_status#27 ASC NULLS FIRST,cd_dep_count#23 ASC NULLS FIRST,cd_dep_employed_count#24 ASC NULLS FIRST,cd_dep_college_count#25 ASC NULLS FIRST], output=[ca_state#19,cd_gender#26,cd_marital_status#27,cd_dep_count#23,cnt1#41,avg(cd_dep_count)#42,max(cd_dep_count)#43,sum(cd_dep_count)#44,cd_dep_employed_count#24,cnt2#45,avg(cd_dep_employed_count)#46,max(cd_dep_employed_count)#47,sum(cd_dep_employed_count)#48,cd_dep_college_count#25,cnt3#49,avg(cd_dep_college_count)#50,max(cd_dep_college_count)#51,sum(cd_dep_college_count)#52]), [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52], 100, 0, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] + +(40) CometColumnarToRow [codegen id : 1] +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/extended.txt new file mode 100644 index 0000000000..42c478c7c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..86604080fc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_datafusion/simplified.txt @@ -0,0 +1,42 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometHashAggregate [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk,ca_state] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk,ca_state] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [customsk] #4 + CometUnion [customsk] + CometProject [ws_bill_customer_sk] [customsk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customsk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/explain.txt new file mode 100644 index 0000000000..ef8880f8db --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/explain.txt @@ -0,0 +1,275 @@ +== Physical Plan == +TakeOrderedAndProject (48) ++- * Project (47) + +- Window (46) + +- * CometColumnarToRow (45) + +- CometSort (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometColumnarExchange (41) + +- * HashAggregate (40) + +- Union (39) + :- * HashAggregate (24) + : +- * CometColumnarToRow (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + :- * HashAggregate (31) + : +- * CometColumnarToRow (30) + : +- CometColumnarExchange (29) + : +- * HashAggregate (28) + : +- * HashAggregate (27) + : +- * CometColumnarToRow (26) + : +- ReusedExchange (25) + +- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * HashAggregate (34) + +- * CometColumnarToRow (33) + +- ReusedExchange (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#6, d_year#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] + +(9) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#8, i_class#9, i_category#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) + +(11) CometProject +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#12] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight + +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((isnotnull(s_state#14) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#14, 2)) = TN)) AND isnotnull(s_store_sk#13)) + +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] + +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] + +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight + +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(21) CometHashAggregate +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] + +(22) CometExchange +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometColumnarToRow [codegen id : 1] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] + +(24) HashAggregate [codegen id : 1] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#17, sum(UnscaledValue(ss_ext_sales_price#3))#18] +Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#17,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2)) as decimal(38,11)) AS gross_margin#19, i_category#12 AS i_category#20, i_class#11 AS i_class#21, 0 AS t_category#22, 0 AS t_class#23, 0 AS lochierarchy#24] + +(25) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#25, i_class#26, sum#27, sum#28] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [i_category#25, i_class#26, sum#27, sum#28] + +(27) HashAggregate [codegen id : 2] +Input [4]: [i_category#25, i_class#26, sum#27, sum#28] +Keys [2]: [i_category#25, i_class#26] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#29))#31, sum(UnscaledValue(ss_ext_sales_price#30))#32] +Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#29))#31,17,2) AS ss_net_profit#33, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#30))#32,17,2) AS ss_ext_sales_price#34, i_category#25] + +(28) HashAggregate [codegen id : 2] +Input [3]: [ss_net_profit#33, ss_ext_sales_price#34, i_category#25] +Keys [1]: [i_category#25] +Functions [2]: [partial_sum(ss_net_profit#33), partial_sum(ss_ext_sales_price#34)] +Aggregate Attributes [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] +Results [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] + +(29) CometColumnarExchange +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] +Arguments: hashpartitioning(i_category#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(30) CometColumnarToRow [codegen id : 3] +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] + +(31) HashAggregate [codegen id : 3] +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] +Keys [1]: [i_category#25] +Functions [2]: [sum(ss_net_profit#33), sum(ss_ext_sales_price#34)] +Aggregate Attributes [2]: [sum(ss_net_profit#33)#43, sum(ss_ext_sales_price#34)#44] +Results [6]: [(sum(ss_net_profit#33)#43 / sum(ss_ext_sales_price#34)#44) AS gross_margin#45, i_category#25, null AS i_class#46, 0 AS t_category#47, 1 AS t_class#48, 1 AS lochierarchy#49] + +(32) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#50, i_class#51, sum#52, sum#53] + +(33) CometColumnarToRow [codegen id : 4] +Input [4]: [i_category#50, i_class#51, sum#52, sum#53] + +(34) HashAggregate [codegen id : 4] +Input [4]: [i_category#50, i_class#51, sum#52, sum#53] +Keys [2]: [i_category#50, i_class#51] +Functions [2]: [sum(UnscaledValue(ss_net_profit#54)), sum(UnscaledValue(ss_ext_sales_price#55))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#54))#31, sum(UnscaledValue(ss_ext_sales_price#55))#32] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#54))#31,17,2) AS ss_net_profit#56, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#55))#32,17,2) AS ss_ext_sales_price#57] + +(35) HashAggregate [codegen id : 4] +Input [2]: [ss_net_profit#56, ss_ext_sales_price#57] +Keys: [] +Functions [2]: [partial_sum(ss_net_profit#56), partial_sum(ss_ext_sales_price#57)] +Aggregate Attributes [4]: [sum#58, isEmpty#59, sum#60, isEmpty#61] +Results [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] + +(36) CometColumnarExchange +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(37) CometColumnarToRow [codegen id : 5] +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] + +(38) HashAggregate [codegen id : 5] +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] +Keys: [] +Functions [2]: [sum(ss_net_profit#56), sum(ss_ext_sales_price#57)] +Aggregate Attributes [2]: [sum(ss_net_profit#56)#66, sum(ss_ext_sales_price#57)#67] +Results [6]: [(sum(ss_net_profit#56)#66 / sum(ss_ext_sales_price#57)#67) AS gross_margin#68, null AS i_category#69, null AS i_class#70, 1 AS t_category#71, 1 AS t_class#72, 2 AS lochierarchy#73] + +(39) Union + +(40) HashAggregate [codegen id : 6] +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Keys [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Functions: [] +Aggregate Attributes: [] +Results [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] + +(41) CometColumnarExchange +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Arguments: hashpartitioning(gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(42) CometHashAggregate +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Keys [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Functions: [] + +(43) CometExchange +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: hashpartitioning(lochierarchy#24, _w0#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(44) CometSort +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74], [lochierarchy#24 ASC NULLS FIRST, _w0#74 ASC NULLS FIRST, gross_margin#19 ASC NULLS FIRST] + +(45) CometColumnarToRow [codegen id : 7] +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] + +(46) Window +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: [rank(gross_margin#19) windowspecdefinition(lochierarchy#24, _w0#74, gross_margin#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#75], [lochierarchy#24, _w0#74], [gross_margin#19 ASC NULLS FIRST] + +(47) Project [codegen id : 8] +Output [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] +Input [6]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74, rank_within_parent#75] + +(48) TakeOrderedAndProject +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] +Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN i_category#20 END ASC NULLS FIRST, rank_within_parent#75 ASC NULLS FIRST], [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/extended.txt new file mode 100644 index 0000000000..e5917cafe9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/extended.txt @@ -0,0 +1,92 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: Unsupported result expressions found in: List(cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#1))#2,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#4,17,2)) as decimal(38,11)) AS gross_margin#5, i_category#6 AS i_category#7, i_class#8 AS i_class#9, 0 AS t_category#10, 0 AS t_class#11, 0 AS lochierarchy#12)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ss_net_profit#13))#14,17,2) AS ss_net_profit#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#16))#17,17,2) AS ss_ext_sales_price#18, i_category#19)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ss_net_profit#20))#14,17,2) AS ss_net_profit#21, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#22))#17,17,2) AS ss_ext_sales_price#23)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 72 out of 84 eligible operators (85%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..4a761aeb47 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_datafusion/simplified.txt @@ -0,0 +1,64 @@ +TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] + WholeStageCodegen (8) + Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] + InputAdapter + Window [gross_margin,lochierarchy,_w0] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [t_category] [gross_margin,i_category,i_class,lochierarchy,_w0,t_class] + CometColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 + WholeStageCodegen (6) + HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class] #3 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #6 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + WholeStageCodegen (3) + HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_category] #7 + WholeStageCodegen (2) + HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum,sum] #3 + WholeStageCodegen (5) + HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #8 + WholeStageCodegen (4) + HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/explain.txt new file mode 100644 index 0000000000..2479e9625c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/explain.txt @@ -0,0 +1,273 @@ +== Physical Plan == +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) + +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight + +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] + +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] + +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] + +(22) HashAggregate [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#17] +Results [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS _w0#19] + +(23) CometColumnarExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] + +(26) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] + +(29) Filter [codegen id : 10] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) + +(30) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] + +(31) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] + +(32) CometColumnarToRow [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] + +(33) HashAggregate [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] +Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] +Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] + +(34) CometColumnarExchange +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(35) CometSort +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] + +(36) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] + +(37) Window +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] + +(39) BroadcastExchange +Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] +Join type: Inner +Join condition: None + +(41) Project [codegen id : 10] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] + +(42) ReusedExchange [Reuses operator id: 34] +Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] + +(43) CometSort +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 8] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] + +(45) Window +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] + +(46) Project [codegen id : 9] +Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] + +(47) BroadcastExchange +Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] + +(48) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] +Join type: Inner +Join condition: None + +(49) Project [codegen id : 10] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] + +(50) TakeOrderedAndProject +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] +Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/extended.txt new file mode 100644 index 0000000000..f63237bcb4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/extended.txt @@ -0,0 +1,93 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#1, i_brand#2, s_store_name#3, s_company_name#4, d_year#5, d_moy#6, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#8,17,2) AS sum_sales#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#8,17,2) AS _w0#10)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#11, i_brand#12, s_store_name#13, s_company_name#14, d_year#15, d_moy#16, MakeDecimal(sum(UnscaledValue(ss_sales_price#17))#8,17,2) AS sum_sales#18)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#11, i_brand#12, s_store_name#13, s_company_name#14, d_year#15, d_moy#16, MakeDecimal(sum(UnscaledValue(ss_sales_price#17))#8,17,2) AS sum_sales#18)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 66 out of 85 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/simplified.txt new file mode 100644 index 0000000000..6a5c95f7a8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_datafusion/simplified.txt @@ -0,0 +1,70 @@ +TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum] + WholeStageCodegen (10) + Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] + BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] + Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] + BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] + Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] + WholeStageCodegen (3) + Filter [d_year] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (6) + Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (9) + Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/explain.txt new file mode 100644 index 0000000000..cba2ffa7be --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/explain.txt @@ -0,0 +1,425 @@ +== Physical Plan == +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales +Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] +PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) + +(3) CometProject +Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] + +(4) CometBroadcastExchange +Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] + +(5) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) + +(7) CometProject +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] + +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft + +(9) CometProject +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] + +(10) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) + +(12) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(13) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] + +(14) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight + +(15) CometProject +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] + +(16) CometHashAggregate +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] +Keys [1]: [ws_item_sk#1] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] + +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Keys [1]: [ws_item_sk#1] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] + +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] + +(22) Window +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] + +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 + +(24) Window +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) + +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] + +(27) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] +PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) + +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(31) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] +PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) + +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(54) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Functions: [] + +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/extended.txt new file mode 100644 index 0000000000..262571ed59 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/extended.txt @@ -0,0 +1,86 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- Filter + +- Window + +- Sort + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 63 out of 80 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/simplified.txt new file mode 100644 index 0000000000..117b74a75b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_datafusion/simplified.txt @@ -0,0 +1,100 @@ +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/explain.txt new file mode 100644 index 0000000000..e447def577 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/explain.txt @@ -0,0 +1,425 @@ +== Physical Plan == +TakeOrderedAndProject (78) ++- * Filter (77) + +- * HashAggregate (76) + +- * HashAggregate (75) + +- * Project (74) + +- * BroadcastHashJoin Inner BuildRight (73) + :- Window (66) + : +- * CometColumnarToRow (65) + : +- CometSort (64) + : +- CometExchange (63) + : +- CometProject (62) + : +- CometFilter (61) + : +- CometSortMergeJoin (60) + : :- CometSort (31) + : : +- CometColumnarExchange (30) + : : +- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- CometColumnarExchange (27) + : : +- * HashAggregate (26) + : : +- * Project (25) + : : +- * BroadcastHashJoin Inner BuildRight (24) + : : :- * Project (17) + : : : +- Window (16) + : : : +- * CometColumnarToRow (15) + : : : +- CometSort (14) + : : : +- CometColumnarExchange (13) + : : : +- * HashAggregate (12) + : : : +- * CometColumnarToRow (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (23) + : : +- * Project (22) + : : +- Window (21) + : : +- * CometColumnarToRow (20) + : : +- CometSort (19) + : : +- ReusedExchange (18) + : +- CometSort (59) + : +- CometColumnarExchange (58) + : +- * HashAggregate (57) + : +- * CometColumnarToRow (56) + : +- CometColumnarExchange (55) + : +- * HashAggregate (54) + : +- * Project (53) + : +- * BroadcastHashJoin Inner BuildRight (52) + : :- * Project (45) + : : +- Window (44) + : : +- * CometColumnarToRow (43) + : : +- CometSort (42) + : : +- CometColumnarExchange (41) + : : +- * HashAggregate (40) + : : +- * CometColumnarToRow (39) + : : +- CometExchange (38) + : : +- CometHashAggregate (37) + : : +- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (33) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (32) + : : +- ReusedExchange (34) + : +- BroadcastExchange (51) + : +- * Project (50) + : +- Window (49) + : +- * CometColumnarToRow (48) + : +- CometSort (47) + : +- ReusedExchange (46) + +- BroadcastExchange (72) + +- * Project (71) + +- Window (70) + +- * CometColumnarToRow (69) + +- CometSort (68) + +- ReusedExchange (67) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] + +(6) CometBroadcastExchange +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] + +(9) CometHashAggregate +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] +Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] + +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(11) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] + +(12) HashAggregate [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] +Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#8] +Results [4]: [ws_item_sk#1 AS item_sk#9, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#8,17,2) AS sumws#10, ws_item_sk#1] + +(13) CometColumnarExchange +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(14) CometSort +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(15) CometColumnarToRow [codegen id : 2] +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] + +(16) Window +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#11], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] + +(17) Project [codegen id : 6] +Output [4]: [item_sk#9, d_date#5, sumws#10, rk#11] +Input [5]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1, rk#11] + +(18) ReusedExchange [Reuses operator id: 13] +Output [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] + +(19) CometSort +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] +Arguments: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15], [ws_item_sk#15 ASC NULLS FIRST, d_date#13 ASC NULLS FIRST] + +(20) CometColumnarToRow [codegen id : 4] +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] + +(21) Window +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] +Arguments: [row_number() windowspecdefinition(ws_item_sk#15, d_date#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#16], [ws_item_sk#15], [d_date#13 ASC NULLS FIRST] + +(22) Project [codegen id : 5] +Output [3]: [item_sk#12, sumws#14, rk#16] +Input [5]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15, rk#16] + +(23) BroadcastExchange +Input [3]: [item_sk#12, sumws#14, rk#16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(24) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [item_sk#9] +Right keys [1]: [item_sk#12] +Join type: Inner +Join condition: (rk#11 >= rk#16) + +(25) Project [codegen id : 6] +Output [4]: [item_sk#9, d_date#5, sumws#10, sumws#14] +Input [7]: [item_sk#9, d_date#5, sumws#10, rk#11, item_sk#12, sumws#14, rk#16] + +(26) HashAggregate [codegen id : 6] +Input [4]: [item_sk#9, d_date#5, sumws#10, sumws#14] +Keys [3]: [item_sk#9, d_date#5, sumws#10] +Functions [1]: [partial_sum(sumws#14)] +Aggregate Attributes [2]: [sum#17, isEmpty#18] +Results [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] + +(27) CometColumnarExchange +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] +Arguments: hashpartitioning(item_sk#9, d_date#5, sumws#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(28) CometColumnarToRow [codegen id : 7] +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] + +(29) HashAggregate [codegen id : 7] +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] +Keys [3]: [item_sk#9, d_date#5, sumws#10] +Functions [1]: [sum(sumws#14)] +Aggregate Attributes [1]: [sum(sumws#14)#21] +Results [3]: [item_sk#9, d_date#5, sum(sumws#14)#21 AS cume_sales#22] + +(30) CometColumnarExchange +Input [3]: [item_sk#9, d_date#5, cume_sales#22] +Arguments: hashpartitioning(item_sk#9, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(31) CometSort +Input [3]: [item_sk#9, d_date#5, cume_sales#22] +Arguments: [item_sk#9, d_date#5, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(32) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#23) + +(34) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#26, d_date#27] + +(35) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Right output [2]: [d_date_sk#26, d_date#27] +Arguments: [ss_sold_date_sk#25], [d_date_sk#26], Inner, BuildRight + +(36) CometProject +Input [5]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25, d_date_sk#26, d_date#27] +Arguments: [ss_item_sk#23, ss_sales_price#24, d_date#27], [ss_item_sk#23, ss_sales_price#24, d_date#27] + +(37) CometHashAggregate +Input [3]: [ss_item_sk#23, ss_sales_price#24, d_date#27] +Keys [2]: [ss_item_sk#23, d_date#27] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#24))] + +(38) CometExchange +Input [3]: [ss_item_sk#23, d_date#27, sum#28] +Arguments: hashpartitioning(ss_item_sk#23, d_date#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometColumnarToRow [codegen id : 8] +Input [3]: [ss_item_sk#23, d_date#27, sum#28] + +(40) HashAggregate [codegen id : 8] +Input [3]: [ss_item_sk#23, d_date#27, sum#28] +Keys [2]: [ss_item_sk#23, d_date#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#24))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#24))#29] +Results [4]: [ss_item_sk#23 AS item_sk#30, d_date#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#24))#29,17,2) AS sumss#31, ss_item_sk#23] + +(41) CometColumnarExchange +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(42) CometSort +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23], [ss_item_sk#23 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST] + +(43) CometColumnarToRow [codegen id : 9] +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] + +(44) Window +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: [row_number() windowspecdefinition(ss_item_sk#23, d_date#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#32], [ss_item_sk#23], [d_date#27 ASC NULLS FIRST] + +(45) Project [codegen id : 13] +Output [4]: [item_sk#30, d_date#27, sumss#31, rk#32] +Input [5]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23, rk#32] + +(46) ReusedExchange [Reuses operator id: 41] +Output [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] + +(47) CometSort +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] +Arguments: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36], [ss_item_sk#36 ASC NULLS FIRST, d_date#34 ASC NULLS FIRST] + +(48) CometColumnarToRow [codegen id : 11] +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] + +(49) Window +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] +Arguments: [row_number() windowspecdefinition(ss_item_sk#36, d_date#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#37], [ss_item_sk#36], [d_date#34 ASC NULLS FIRST] + +(50) Project [codegen id : 12] +Output [3]: [item_sk#33, sumss#35, rk#37] +Input [5]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36, rk#37] + +(51) BroadcastExchange +Input [3]: [item_sk#33, sumss#35, rk#37] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] + +(52) BroadcastHashJoin [codegen id : 13] +Left keys [1]: [item_sk#30] +Right keys [1]: [item_sk#33] +Join type: Inner +Join condition: (rk#32 >= rk#37) + +(53) Project [codegen id : 13] +Output [4]: [item_sk#30, d_date#27, sumss#31, sumss#35] +Input [7]: [item_sk#30, d_date#27, sumss#31, rk#32, item_sk#33, sumss#35, rk#37] + +(54) HashAggregate [codegen id : 13] +Input [4]: [item_sk#30, d_date#27, sumss#31, sumss#35] +Keys [3]: [item_sk#30, d_date#27, sumss#31] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#38, isEmpty#39] +Results [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] + +(55) CometColumnarExchange +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] +Arguments: hashpartitioning(item_sk#30, d_date#27, sumss#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(56) CometColumnarToRow [codegen id : 14] +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] + +(57) HashAggregate [codegen id : 14] +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] +Keys [3]: [item_sk#30, d_date#27, sumss#31] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#42] +Results [3]: [item_sk#30, d_date#27, sum(sumss#35)#42 AS cume_sales#43] + +(58) CometColumnarExchange +Input [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: hashpartitioning(item_sk#30, d_date#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(59) CometSort +Input [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#30, d_date#27, cume_sales#43], [item_sk#30 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST] + +(60) CometSortMergeJoin +Left output [3]: [item_sk#9, d_date#5, cume_sales#22] +Right output [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#9, d_date#5], [item_sk#30, d_date#27], FullOuter + +(61) CometFilter +Input [6]: [item_sk#9, d_date#5, cume_sales#22, item_sk#30, d_date#27, cume_sales#43] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END) + +(62) CometProject +Input [6]: [item_sk#9, d_date#5, cume_sales#22, item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#27 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] + +(63) CometExchange +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(64) CometSort +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] + +(65) CometColumnarToRow [codegen id : 15] +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] + +(66) Window +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [item_sk#44], [d_date#45 ASC NULLS FIRST] + +(67) ReusedExchange [Reuses operator id: 63] +Output [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] + +(68) CometSort +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] +Arguments: [item_sk#49, d_date#50, web_sales#51, store_sales#52], [item_sk#49 ASC NULLS FIRST, d_date#50 ASC NULLS FIRST] + +(69) CometColumnarToRow [codegen id : 30] +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] + +(70) Window +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] +Arguments: [row_number() windowspecdefinition(item_sk#49, d_date#50 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#53], [item_sk#49], [d_date#50 ASC NULLS FIRST] + +(71) Project [codegen id : 31] +Output [4]: [item_sk#49, web_sales#51, store_sales#52, rk#53] +Input [5]: [item_sk#49, d_date#50, web_sales#51, store_sales#52, rk#53] + +(72) BroadcastExchange +Input [4]: [item_sk#49, web_sales#51, store_sales#52, rk#53] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(73) BroadcastHashJoin [codegen id : 32] +Left keys [1]: [item_sk#44] +Right keys [1]: [item_sk#49] +Join type: Inner +Join condition: (rk#48 >= rk#53) + +(74) Project [codegen id : 32] +Output [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] +Input [9]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#48, item_sk#49, web_sales#51, store_sales#52, rk#53] + +(75) HashAggregate [codegen id : 32] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] +Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Functions [2]: [partial_max(web_sales#51), partial_max(store_sales#52)] +Aggregate Attributes [2]: [max#54, max#55] +Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#56, max#57] + +(76) HashAggregate [codegen id : 32] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#56, max#57] +Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Functions [2]: [max(web_sales#51), max(store_sales#52)] +Aggregate Attributes [2]: [max(web_sales#51)#58, max(store_sales#52)#59] +Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max(web_sales#51)#58 AS web_cumulative#60, max(store_sales#52)#59 AS store_cumulative#61] + +(77) Filter [codegen id : 32] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] +Condition : ((isnotnull(web_cumulative#60) AND isnotnull(store_cumulative#61)) AND (web_cumulative#60 > store_cumulative#61)) + +(78) TakeOrderedAndProject +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] +Arguments: 100, [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/extended.txt new file mode 100644 index 0000000000..d5b42a3a51 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/extended.txt @@ -0,0 +1,196 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- HashAggregate + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- CometColumnarToRow + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometColumnarExchange + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ws_item_sk#1 AS item_sk#2, d_date#3, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#5,17,2) AS sumws#6, ws_item_sk#1)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ws_item_sk#1 AS item_sk#2, d_date#3, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#5,17,2) AS sumws#6, ws_item_sk#1)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_item_sk#7 AS item_sk#8, d_date#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#11,17,2) AS sumss#12, ss_item_sk#7)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_item_sk#7 AS item_sk#8, d_date#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#11,17,2) AS sumss#12, ss_item_sk#7)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ws_item_sk#1 AS item_sk#2, d_date#3, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#5,17,2) AS sumws#6, ws_item_sk#1)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ws_item_sk#1 AS item_sk#2, d_date#3, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#5,17,2) AS sumws#6, ws_item_sk#1)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_item_sk#7 AS item_sk#8, d_date#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#11,17,2) AS sumss#12, ss_item_sk#7)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(ss_item_sk#7 AS item_sk#8, d_date#9, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#11,17,2) AS sumss#12, ss_item_sk#7)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 118 out of 172 eligible operators (68%). Final plan contains 22 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..711e542481 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_datafusion/simplified.txt @@ -0,0 +1,113 @@ +TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] + WholeStageCodegen (32) + Filter [web_cumulative,store_cumulative] + HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] + HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] + Project [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + InputAdapter + Window [item_sk,d_date] + WholeStageCodegen (15) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (7) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumws] #3 + WholeStageCodegen (6) + HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumws,sumws] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + CometColumnarExchange [ws_item_sk] #4 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum] + CometColumnarToRow + InputAdapter + CometExchange [ws_item_sk,d_date] #5 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (5) + Project [item_sk,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #8 + WholeStageCodegen (14) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumss] #9 + WholeStageCodegen (13) + HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumss,sumss] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (9) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + CometColumnarExchange [ss_item_sk] #10 + WholeStageCodegen (8) + HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_item_sk,d_date] #11 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #6 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (12) + Project [item_sk,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (31) + Project [item_sk,web_sales,store_sales,rk] + InputAdapter + Window [item_sk,d_date] + WholeStageCodegen (30) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,web_sales,store_sales] + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/explain.txt new file mode 100644 index 0000000000..46244f48e6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/explain.txt @@ -0,0 +1,273 @@ +== Physical Plan == +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] + +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight + +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] + +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] + +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] + +(22) HashAggregate [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#16] +Results [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS _w0#18] + +(23) CometColumnarExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] + +(26) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#14, d_year#11] + +(29) Filter [codegen id : 10] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) + +(30) Project [codegen id : 10] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(31) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] + +(32) CometColumnarToRow [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] + +(33) HashAggregate [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] +Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] +Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] +Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] + +(34) CometColumnarExchange +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(35) CometSort +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] + +(36) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] + +(37) Window +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] + +(39) BroadcastExchange +Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] +Join type: Inner +Join condition: None + +(41) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] + +(42) ReusedExchange [Reuses operator id: 34] +Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] + +(43) CometSort +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 8] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] + +(45) Window +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] + +(46) Project [codegen id : 9] +Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] + +(47) BroadcastExchange +Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] + +(48) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] +Join type: Inner +Join condition: None + +(49) Project [codegen id : 10] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] + +(50) TakeOrderedAndProject +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/extended.txt new file mode 100644 index 0000000000..7e041c6501 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/extended.txt @@ -0,0 +1,93 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#1, i_brand#2, cc_name#3, d_year#4, d_moy#5, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#7,17,2) AS sum_sales#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#7,17,2) AS _w0#9)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.call_center + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#10, i_brand#11, cc_name#12, d_year#13, d_moy#14, MakeDecimal(sum(UnscaledValue(cs_sales_price#15))#7,17,2) AS sum_sales#16)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_category#10, i_brand#11, cc_name#12, d_year#13, d_moy#14, MakeDecimal(sum(UnscaledValue(cs_sales_price#15))#7,17,2) AS sum_sales#16)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 66 out of 85 eligible operators (77%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/simplified.txt new file mode 100644 index 0000000000..52e3b94683 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_datafusion/simplified.txt @@ -0,0 +1,70 @@ +TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum] + WholeStageCodegen (10) + Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] + BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] + Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] + BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] + Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_category,i_brand,cc_name,d_year] + WholeStageCodegen (3) + Filter [d_year] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,cc_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (6) + Project [i_category,i_brand,cc_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,cc_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (9) + Project [i_category,i_brand,cc_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/explain.txt new file mode 100644 index 0000000000..bca4a132b6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/explain.txt @@ -0,0 +1,531 @@ +== Physical Plan == +* CometColumnarToRow (92) ++- CometTakeOrderedAndProject (91) + +- CometHashAggregate (90) + +- CometColumnarExchange (89) + +- * HashAggregate (88) + +- Union (87) + :- * HashAggregate (72) + : +- * CometColumnarToRow (71) + : +- CometColumnarExchange (70) + : +- * HashAggregate (69) + : +- Union (68) + : :- * HashAggregate (23) + : : +- * CometColumnarToRow (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : :- * HashAggregate (43) + : : +- * CometColumnarToRow (42) + : : +- CometExchange (41) + : : +- CometHashAggregate (40) + : : +- CometProject (39) + : : +- CometBroadcastHashJoin (38) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometUnion (30) + : : : : :- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (24) + : : : : +- CometProject (29) + : : : : +- CometFilter (28) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (27) + : : : +- ReusedExchange (31) + : : +- CometBroadcastExchange (37) + : : +- CometProject (36) + : : +- CometFilter (35) + : : +- CometNativeScan parquet spark_catalog.default.catalog_page (34) + : +- * HashAggregate (67) + : +- * CometColumnarToRow (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometUnion (54) + : : : :- CometProject (46) + : : : : +- CometFilter (45) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (44) + : : : +- CometProject (53) + : : : +- CometBroadcastHashJoin (52) + : : : :- CometBroadcastExchange (48) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (47) + : : : +- CometProject (51) + : : : +- CometFilter (50) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (49) + : : +- ReusedExchange (55) + : +- CometBroadcastExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometNativeScan parquet spark_catalog.default.web_site (58) + :- * HashAggregate (79) + : +- * CometColumnarToRow (78) + : +- CometColumnarExchange (77) + : +- * HashAggregate (76) + : +- * HashAggregate (75) + : +- * CometColumnarToRow (74) + : +- ReusedExchange (73) + +- * HashAggregate (86) + +- * CometColumnarToRow (85) + +- CometColumnarExchange (84) + +- * HashAggregate (83) + +- * HashAggregate (82) + +- * CometColumnarToRow (81) + +- ReusedExchange (80) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) + +(3) CometProject +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] + +(4) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) + +(6) CometProject +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] + +(7) CometUnion +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] + +(8) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_date#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) + +(10) CometProject +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(11) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(12) CometBroadcastHashJoin +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] + +(14) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#23, s_store_id#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) + +(16) CometProject +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#25] + +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight + +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] + +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] + +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] + +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#25) AS id#35, MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#36, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#38] + +(24) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +PushedFilters: [IsNotNull(cs_catalog_page_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) + +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] + +(27) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] +PushedFilters: [IsNotNull(cr_catalog_page_sk)] +ReadSchema: struct + +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) + +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] + +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] + +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] + +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight + +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] + +(34) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#61, 16)) AS cp_catalog_page_id#62] + +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] + +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight + +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] + +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] + +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] + +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#62) AS id#72, MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#75] + +(44) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] +PushedFilters: [IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) + +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] + +(47) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] +ReadSchema: struct + +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] + +(49) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) + +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] + +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft + +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] + +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] + +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] + +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight + +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] + +(58) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) + +(60) CometProject +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#103, 16)) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [web channel AS channel#113, concat(web_site, web_site_id#104) AS id#114, MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#115, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#116, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#117] + +(68) Union + +(69) HashAggregate [codegen id : 4] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(70) CometColumnarExchange +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(71) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(72) HashAggregate [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] + +(73) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] + +(74) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] + +(75) HashAggregate [codegen id : 10] +Input [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +Keys [2]: [channel#136, id#137] +Functions [3]: [sum(sales#144), sum(returns#145), sum(profit#146)] +Aggregate Attributes [3]: [sum(sales#144)#130, sum(returns#145)#131, sum(profit#146)#132] +Results [4]: [channel#136, sum(sales#144)#130 AS sales#147, sum(returns#145)#131 AS returns#148, sum(profit#146)#132 AS profit#149] + +(76) HashAggregate [codegen id : 10] +Input [4]: [channel#136, sales#147, returns#148, profit#149] +Keys [1]: [channel#136] +Functions [3]: [partial_sum(sales#147), partial_sum(returns#148), partial_sum(profit#149)] +Aggregate Attributes [6]: [sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] +Results [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] + +(77) CometColumnarExchange +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Arguments: hashpartitioning(channel#136, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(78) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] + +(79) HashAggregate [codegen id : 11] +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Keys [1]: [channel#136] +Functions [3]: [sum(sales#147), sum(returns#148), sum(profit#149)] +Aggregate Attributes [3]: [sum(sales#147)#162, sum(returns#148)#163, sum(profit#149)#164] +Results [5]: [channel#136, null AS id#165, sum(sales#147)#162 AS sum(sales)#166, sum(returns#148)#163 AS sum(returns)#167, sum(profit#149)#164 AS sum(profit)#168] + +(80) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] + +(81) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] + +(82) HashAggregate [codegen id : 16] +Input [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Keys [2]: [channel#169, id#170] +Functions [3]: [sum(sales#177), sum(returns#178), sum(profit#179)] +Aggregate Attributes [3]: [sum(sales#177)#130, sum(returns#178)#131, sum(profit#179)#132] +Results [3]: [sum(sales#177)#130 AS sales#180, sum(returns#178)#131 AS returns#181, sum(profit#179)#132 AS profit#182] + +(83) HashAggregate [codegen id : 16] +Input [3]: [sales#180, returns#181, profit#182] +Keys: [] +Functions [3]: [partial_sum(sales#180), partial_sum(returns#181), partial_sum(profit#182)] +Aggregate Attributes [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] +Results [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] + +(84) CometColumnarExchange +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(85) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] + +(86) HashAggregate [codegen id : 17] +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] +Keys: [] +Functions [3]: [sum(sales#180), sum(returns#181), sum(profit#182)] +Aggregate Attributes [3]: [sum(sales#180)#195, sum(returns#181)#196, sum(profit#182)#197] +Results [5]: [null AS channel#198, null AS id#199, sum(sales#180)#195 AS sum(sales)#200, sum(returns#181)#196 AS sum(returns)#201, sum(profit#182)#197 AS sum(profit)#202] + +(87) Union + +(88) HashAggregate [codegen id : 18] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] + +(89) CometColumnarExchange +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(90) CometHashAggregate +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Functions: [] + +(91) CometTakeOrderedAndProject +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, 0, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] + +(92) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/extended.txt new file mode 100644 index 0000000000..cf0af2a841 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/extended.txt @@ -0,0 +1,250 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(store channel AS channel#1, concat(store, s_store_id#2) AS id#3, MakeDecimal(sum(UnscaledValue(sales_price#4))#5,17,2) AS sales#6, MakeDecimal(sum(UnscaledValue(return_amt#7))#8,17,2) AS returns#9, (MakeDecimal(sum(UnscaledValue(profit#10))#11,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#12))#13,17,2)) AS profit#14)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(catalog channel AS channel#15, concat(catalog_page, cp_catalog_page_id#16) AS id#17, MakeDecimal(sum(UnscaledValue(sales_price#18))#19,17,2) AS sales#20, MakeDecimal(sum(UnscaledValue(return_amt#21))#22,17,2) AS returns#23, (MakeDecimal(sum(UnscaledValue(profit#24))#25,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#26))#27,17,2)) AS profit#28)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(web channel AS channel#29, concat(web_site, web_site_id#30) AS id#31, MakeDecimal(sum(UnscaledValue(sales_price#32))#33,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#35))#36,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#38))#39,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#40))#41,17,2)) AS profit#42)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastExchange + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(store channel AS channel#1, concat(store, s_store_id#2) AS id#3, MakeDecimal(sum(UnscaledValue(sales_price#4))#5,17,2) AS sales#6, MakeDecimal(sum(UnscaledValue(return_amt#7))#8,17,2) AS returns#9, (MakeDecimal(sum(UnscaledValue(profit#10))#11,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#12))#13,17,2)) AS profit#14)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(catalog channel AS channel#15, concat(catalog_page, cp_catalog_page_id#16) AS id#17, MakeDecimal(sum(UnscaledValue(sales_price#18))#19,17,2) AS sales#20, MakeDecimal(sum(UnscaledValue(return_amt#21))#22,17,2) AS returns#23, (MakeDecimal(sum(UnscaledValue(profit#24))#25,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#26))#27,17,2)) AS profit#28)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(web channel AS channel#29, concat(web_site, web_site_id#30) AS id#31, MakeDecimal(sum(UnscaledValue(sales_price#32))#33,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#35))#36,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#38))#39,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#40))#41,17,2)) AS profit#42)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastExchange + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(store channel AS channel#1, concat(store, s_store_id#2) AS id#3, MakeDecimal(sum(UnscaledValue(sales_price#4))#5,17,2) AS sales#6, MakeDecimal(sum(UnscaledValue(return_amt#7))#8,17,2) AS returns#9, (MakeDecimal(sum(UnscaledValue(profit#10))#11,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#12))#13,17,2)) AS profit#14)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(catalog channel AS channel#15, concat(catalog_page, cp_catalog_page_id#16) AS id#17, MakeDecimal(sum(UnscaledValue(sales_price#18))#19,17,2) AS sales#20, MakeDecimal(sum(UnscaledValue(return_amt#21))#22,17,2) AS returns#23, (MakeDecimal(sum(UnscaledValue(profit#24))#25,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#26))#27,17,2)) AS profit#28)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_page + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(web channel AS channel#29, concat(web_site, web_site_id#30) AS id#31, MakeDecimal(sum(UnscaledValue(sales_price#32))#33,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#35))#36,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#38))#39,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#40))#41,17,2)) AS profit#42)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 209 out of 233 eligible operators (89%). Final plan contains 15 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..383183f1df --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_datafusion/simplified.txt @@ -0,0 +1,114 @@ +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #8 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #11 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #12 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/explain.txt new file mode 100644 index 0000000000..ad6f6c0aab --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/explain.txt @@ -0,0 +1,1013 @@ +== Physical Plan == +* CometColumnarToRow (183) ++- CometSort (182) + +- CometExchange (181) + +- CometProject (180) + +- CometSortMergeJoin (179) + :- CometSort (115) + : +- CometColumnarExchange (114) + : +- * HashAggregate (113) + : +- * HashAggregate (112) + : +- * Project (111) + : +- * BroadcastHashJoin Inner BuildRight (110) + : :- * Project (104) + : : +- * BroadcastHashJoin Inner BuildRight (103) + : : :- * Project (101) + : : : +- * BroadcastHashJoin Inner BuildRight (100) + : : : :- * Project (95) + : : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : : :- * Project (92) + : : : : : +- * BroadcastHashJoin Inner BuildRight (91) + : : : : : :- * Project (85) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) + : : : : : : :- * Project (82) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) + : : : : : : : :- * Project (76) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) + : : : : : : : : :- * Project (70) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) + : : : : : : : : : :- * Project (67) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) + : : : : : : : : : : :- * Project (60) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (59) + : : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : : :- * Project (51) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) + : : : : : : : : : : : : : :- * Project (45) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (44) + : : : : : : : : : : : : : : :- * Project (38) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (37) + : : : : : : : : : : : : : : : :- * Project (32) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) + : : : : : : : : : : : : : : : : :- * CometColumnarToRow (11) + : : : : : : : : : : : : : : : : : +- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- * Sort (30) + : : : : : : : : : : : : : : : : +- * Project (29) + : : : : : : : : : : : : : : : : +- * Filter (28) + : : : : : : : : : : : : : : : : +- * HashAggregate (27) + : : : : : : : : : : : : : : : : +- * CometColumnarToRow (26) + : : : : : : : : : : : : : : : : +- CometExchange (25) + : : : : : : : : : : : : : : : : +- CometHashAggregate (24) + : : : : : : : : : : : : : : : : +- CometProject (23) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) + : : : : : : : : : : : : : : : : :- CometSort (16) + : : : : : : : : : : : : : : : : : +- CometExchange (15) + : : : : : : : : : : : : : : : : : +- CometProject (14) + : : : : : : : : : : : : : : : : : +- CometFilter (13) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (12) + : : : : : : : : : : : : : : : : +- CometSort (21) + : : : : : : : : : : : : : : : : +- CometExchange (20) + : : : : : : : : : : : : : : : : +- CometProject (19) + : : : : : : : : : : : : : : : : +- CometFilter (18) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (17) + : : : : : : : : : : : : : : : +- BroadcastExchange (36) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (35) + : : : : : : : : : : : : : : : +- CometFilter (34) + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (33) + : : : : : : : : : : : : : : +- BroadcastExchange (43) + : : : : : : : : : : : : : : +- * CometColumnarToRow (42) + : : : : : : : : : : : : : : +- CometProject (41) + : : : : : : : : : : : : : : +- CometFilter (40) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (39) + : : : : : : : : : : : : : +- BroadcastExchange (49) + : : : : : : : : : : : : : +- * CometColumnarToRow (48) + : : : : : : : : : : : : : +- CometFilter (47) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (46) + : : : : : : : : : : : : +- BroadcastExchange (55) + : : : : : : : : : : : : +- * CometColumnarToRow (54) + : : : : : : : : : : : : +- CometFilter (53) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (52) + : : : : : : : : : : : +- ReusedExchange (58) + : : : : : : : : : : +- BroadcastExchange (65) + : : : : : : : : : : +- * CometColumnarToRow (64) + : : : : : : : : : : +- CometProject (63) + : : : : : : : : : : +- CometFilter (62) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (61) + : : : : : : : : : +- ReusedExchange (68) + : : : : : : : : +- BroadcastExchange (74) + : : : : : : : : +- * CometColumnarToRow (73) + : : : : : : : : +- CometFilter (72) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (71) + : : : : : : : +- BroadcastExchange (80) + : : : : : : : +- * CometColumnarToRow (79) + : : : : : : : +- CometFilter (78) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (77) + : : : : : : +- ReusedExchange (83) + : : : : : +- BroadcastExchange (90) + : : : : : +- * CometColumnarToRow (89) + : : : : : +- CometProject (88) + : : : : : +- CometFilter (87) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (86) + : : : : +- ReusedExchange (93) + : : : +- BroadcastExchange (99) + : : : +- * CometColumnarToRow (98) + : : : +- CometFilter (97) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (96) + : : +- ReusedExchange (102) + : +- BroadcastExchange (109) + : +- * CometColumnarToRow (108) + : +- CometProject (107) + : +- CometFilter (106) + : +- CometNativeScan parquet spark_catalog.default.item (105) + +- CometSort (178) + +- CometColumnarExchange (177) + +- * HashAggregate (176) + +- * HashAggregate (175) + +- * Project (174) + +- * BroadcastHashJoin Inner BuildRight (173) + :- * Project (171) + : +- * BroadcastHashJoin Inner BuildRight (170) + : :- * Project (168) + : : +- * BroadcastHashJoin Inner BuildRight (167) + : : :- * Project (165) + : : : +- * BroadcastHashJoin Inner BuildRight (164) + : : : :- * Project (162) + : : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : : :- * Project (159) + : : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : : :- * Project (156) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : : :- * Project (153) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : : :- * Project (150) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : : :- * Project (147) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : : :- * Project (144) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : : :- * Project (141) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : : :- * Project (126) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (125) + : : : : : : : : : : : : : : : :- * CometColumnarToRow (118) + : : : : : : : : : : : : : : : : +- CometSort (117) + : : : : : : : : : : : : : : : : +- ReusedExchange (116) + : : : : : : : : : : : : : : : +- * Sort (124) + : : : : : : : : : : : : : : : +- * Project (123) + : : : : : : : : : : : : : : : +- * Filter (122) + : : : : : : : : : : : : : : : +- * HashAggregate (121) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- BroadcastExchange (130) + : : : : : : : : : : : : : : +- * CometColumnarToRow (129) + : : : : : : : : : : : : : : +- CometFilter (128) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (127) + : : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : : +- ReusedExchange (145) + : : : : : : : : +- ReusedExchange (148) + : : : : : : : +- ReusedExchange (151) + : : : : : : +- ReusedExchange (154) + : : : : : +- ReusedExchange (157) + : : : : +- ReusedExchange (160) + : : : +- ReusedExchange (163) + : : +- ReusedExchange (166) + : +- ReusedExchange (169) + +- ReusedExchange (172) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] +ReadSchema: struct + +(2) CometFilter +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) + +(3) CometBroadcastExchange +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(4) CometNativeScan parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) + +(6) CometProject +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] + +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft + +(8) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(9) CometExchange +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(10) CometSort +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] + +(11) CometColumnarToRow [codegen id : 1] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(12) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct + +(13) CometFilter +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) + +(14) CometProject +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] + +(15) CometExchange +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(16) CometSort +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] + +(17) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(18) CometFilter +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) + +(19) CometProject +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] + +(20) CometExchange +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(21) CometSort +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] + +(22) CometSortMergeJoin +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner + +(23) CometProject +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] + +(24) CometHashAggregate +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] + +(25) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] + +(27) HashAggregate [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#29, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30] +Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#29,17,2) AS sale#31, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30 AS refund#32] + +(28) Filter [codegen id : 2] +Input [3]: [cs_item_sk#16, sale#31, refund#32] +Condition : ((isnotnull(sale#31) AND isnotnull(refund#32)) AND (cast(sale#31 as decimal(21,2)) > (2 * refund#32))) + +(29) Project [codegen id : 2] +Output [1]: [cs_item_sk#16] +Input [3]: [cs_item_sk#16, sale#31, refund#32] + +(30) Sort [codegen id : 2] +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 + +(31) SortMergeJoin [codegen id : 18] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [cs_item_sk#16] +Join type: Inner +Join condition: None + +(32) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] + +(33) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#33, d_year#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [d_date_sk#33, d_year#34] +Condition : ((isnotnull(d_year#34) AND (d_year#34 = 1999)) AND isnotnull(d_date_sk#33)) + +(35) CometColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#33, d_year#34] + +(36) BroadcastExchange +Input [2]: [d_date_sk#33, d_year#34] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] + +(37) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_sold_date_sk#12] +Right keys [1]: [d_date_sk#33] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#33, d_year#34] + +(39) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Condition : ((isnotnull(s_store_sk#35) AND isnotnull(s_store_name#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)))) + +(41) CometProject +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Arguments: [s_store_sk#35, s_store_name#36, s_zip#38], [s_store_sk#35, s_store_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)) AS s_zip#38] + +(42) CometColumnarToRow [codegen id : 4] +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] + +(43) BroadcastExchange +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] + +(44) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_store_sk#6] +Right keys [1]: [s_store_sk#35] +Join type: Inner +Join condition: None + +(45) Project [codegen id : 18] +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_sk#35, s_store_name#36, s_zip#38] + +(46) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(47) CometFilter +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Condition : (((((isnotnull(c_customer_sk#39) AND isnotnull(c_first_sales_date_sk#44)) AND isnotnull(c_first_shipto_date_sk#43)) AND isnotnull(c_current_cdemo_sk#40)) AND isnotnull(c_current_hdemo_sk#41)) AND isnotnull(c_current_addr_sk#42)) + +(48) CometColumnarToRow [codegen id : 5] +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] + +(49) BroadcastExchange +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] + +(50) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_customer_sk#2] +Right keys [1]: [c_customer_sk#39] +Join type: Inner +Join condition: None + +(51) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] + +(52) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#45, d_year#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(53) CometFilter +Input [2]: [d_date_sk#45, d_year#46] +Condition : isnotnull(d_date_sk#45) + +(54) CometColumnarToRow [codegen id : 6] +Input [2]: [d_date_sk#45, d_year#46] + +(55) BroadcastExchange +Input [2]: [d_date_sk#45, d_year#46] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] + +(56) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_sales_date_sk#44] +Right keys [1]: [d_date_sk#45] +Join type: Inner +Join condition: None + +(57) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44, d_date_sk#45, d_year#46] + +(58) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#47, d_year#48] + +(59) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_shipto_date_sk#43] +Right keys [1]: [d_date_sk#47] +Join type: Inner +Join condition: None + +(60) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46, d_date_sk#47, d_year#48] + +(61) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#49, cd_marital_status#50] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(62) CometFilter +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Condition : (isnotnull(cd_demo_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)))) + +(63) CometProject +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Arguments: [cd_demo_sk#49, cd_marital_status#51], [cd_demo_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)) AS cd_marital_status#51] + +(64) CometColumnarToRow [codegen id : 8] +Input [2]: [cd_demo_sk#49, cd_marital_status#51] + +(65) BroadcastExchange +Input [2]: [cd_demo_sk#49, cd_marital_status#51] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(66) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_cdemo_sk#3] +Right keys [1]: [cd_demo_sk#49] +Join type: Inner +Join condition: None + +(67) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_demo_sk#49, cd_marital_status#51] + +(68) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#52, cd_marital_status#53] + +(69) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_cdemo_sk#40] +Right keys [1]: [cd_demo_sk#52] +Join type: Inner +Join condition: NOT (cd_marital_status#51 = cd_marital_status#53) + +(70) Project [codegen id : 18] +Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51, cd_demo_sk#52, cd_marital_status#53] + +(71) CometNativeScan parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(72) CometFilter +Input [1]: [p_promo_sk#54] +Condition : isnotnull(p_promo_sk#54) + +(73) CometColumnarToRow [codegen id : 10] +Input [1]: [p_promo_sk#54] + +(74) BroadcastExchange +Input [1]: [p_promo_sk#54] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] + +(75) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_promo_sk#7] +Right keys [1]: [p_promo_sk#54] +Join type: Inner +Join condition: None + +(76) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, p_promo_sk#54] + +(77) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct + +(78) CometFilter +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Condition : (isnotnull(hd_demo_sk#55) AND isnotnull(hd_income_band_sk#56)) + +(79) CometColumnarToRow [codegen id : 11] +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] + +(80) BroadcastExchange +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] + +(81) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_hdemo_sk#4] +Right keys [1]: [hd_demo_sk#55] +Join type: Inner +Join condition: None + +(82) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_demo_sk#55, hd_income_band_sk#56] + +(83) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] + +(84) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_hdemo_sk#41] +Right keys [1]: [hd_demo_sk#57] +Join type: Inner +Join condition: None + +(85) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_demo_sk#57, hd_income_band_sk#58] + +(86) CometNativeScan parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(87) CometFilter +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Condition : isnotnull(ca_address_sk#59) + +(88) CometProject +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Arguments: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65], [ca_address_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#60, 10)) AS ca_street_number#64, ca_street_name#61, ca_city#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#63, 10)) AS ca_zip#65] + +(89) CometColumnarToRow [codegen id : 13] +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] + +(90) BroadcastExchange +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(91) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_addr_sk#5] +Right keys [1]: [ca_address_sk#59] +Join type: Inner +Join condition: None + +(92) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] + +(93) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] + +(94) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_addr_sk#42] +Right keys [1]: [ca_address_sk#66] +Join type: Inner +Join condition: None + +(95) Project [codegen id : 18] +Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] + +(96) CometNativeScan parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#71] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(97) CometFilter +Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) + +(98) CometColumnarToRow [codegen id : 15] +Input [1]: [ib_income_band_sk#71] + +(99) BroadcastExchange +Input [1]: [ib_income_band_sk#71] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] + +(100) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#56] +Right keys [1]: [ib_income_band_sk#71] +Join type: Inner +Join condition: None + +(101) Project [codegen id : 18] +Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] + +(102) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#72] + +(103) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#58] +Right keys [1]: [ib_income_band_sk#72] +Join type: Inner +Join condition: None + +(104) Project [codegen id : 18] +Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] + +(105) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(106) CometFilter +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#75, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) + +(107) CometProject +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#77], [i_item_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#76, 50)) AS i_product_name#77] + +(108) CometColumnarToRow [codegen id : 17] +Input [2]: [i_item_sk#73, i_product_name#77] + +(109) BroadcastExchange +Input [2]: [i_item_sk#73, i_product_name#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] + +(110) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [i_item_sk#73] +Join type: Inner +Join condition: None + +(111) Project [codegen id : 18] +Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] + +(112) HashAggregate [codegen id : 18] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] +Aggregate Attributes [4]: [count#78, sum#79, sum#80, sum#81] +Results [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] + +(113) HashAggregate [codegen id : 18] +Input [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#9))#87, sum(UnscaledValue(ss_list_price#10))#88, sum(UnscaledValue(ss_coupon_amt#11))#89] +Results [17]: [i_product_name#77 AS product_name#90, i_item_sk#73 AS item_sk#91, s_store_name#36 AS store_name#92, s_zip#38 AS store_zip#93, ca_street_number#64 AS b_street_number#94, ca_street_name#61 AS b_streen_name#95, ca_city#62 AS b_city#96, ca_zip#65 AS b_zip#97, ca_street_number#67 AS c_street_number#98, ca_street_name#68 AS c_street_name#99, ca_city#69 AS c_city#100, ca_zip#70 AS c_zip#101, d_year#34 AS syear#102, count(1)#86 AS cnt#103, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#87,17,2) AS s1#104, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#88,17,2) AS s2#105, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#89,17,2) AS s3#106] + +(114) CometColumnarExchange +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: hashpartitioning(item_sk#91, store_name#92, store_zip#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] + +(115) CometSort +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106], [item_sk#91 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, store_zip#93 ASC NULLS FIRST] + +(116) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(117) CometSort +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Arguments: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#107 ASC NULLS FIRST] + +(118) CometColumnarToRow [codegen id : 19] +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(119) ReusedExchange [Reuses operator id: 25] +Output [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] + +(120) CometColumnarToRow [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] + +(121) HashAggregate [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] +Keys [1]: [cs_item_sk#118] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#122)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#122))#29, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30] +Results [3]: [cs_item_sk#118, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#122))#29,17,2) AS sale#126, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30 AS refund#127] + +(122) Filter [codegen id : 20] +Input [3]: [cs_item_sk#118, sale#126, refund#127] +Condition : ((isnotnull(sale#126) AND isnotnull(refund#127)) AND (cast(sale#126 as decimal(21,2)) > (2 * refund#127))) + +(123) Project [codegen id : 20] +Output [1]: [cs_item_sk#118] +Input [3]: [cs_item_sk#118, sale#126, refund#127] + +(124) Sort [codegen id : 20] +Input [1]: [cs_item_sk#118] +Arguments: [cs_item_sk#118 ASC NULLS FIRST], false, 0 + +(125) SortMergeJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [cs_item_sk#118] +Join type: Inner +Join condition: None + +(126) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Input [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#118] + +(127) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#128, d_year#129] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(128) CometFilter +Input [2]: [d_date_sk#128, d_year#129] +Condition : ((isnotnull(d_year#129) AND (d_year#129 = 2000)) AND isnotnull(d_date_sk#128)) + +(129) CometColumnarToRow [codegen id : 21] +Input [2]: [d_date_sk#128, d_year#129] + +(130) BroadcastExchange +Input [2]: [d_date_sk#128, d_year#129] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] + +(131) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_sold_date_sk#117] +Right keys [1]: [d_date_sk#128] +Join type: Inner +Join condition: None + +(132) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129] +Input [13]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#128, d_year#129] + +(133) ReusedExchange [Reuses operator id: 43] +Output [3]: [s_store_sk#130, s_store_name#131, s_zip#132] + +(134) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_store_sk#112] +Right keys [1]: [s_store_sk#130] +Join type: Inner +Join condition: None + +(135) Project [codegen id : 36] +Output [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132] +Input [14]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_sk#130, s_store_name#131, s_zip#132] + +(136) ReusedExchange [Reuses operator id: 49] +Output [6]: [c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] + +(137) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_customer_sk#108] +Right keys [1]: [c_customer_sk#133] +Join type: Inner +Join condition: None + +(138) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] +Input [18]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] + +(139) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#139, d_year#140] + +(140) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_sales_date_sk#138] +Right keys [1]: [d_date_sk#139] +Join type: Inner +Join condition: None + +(141) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138, d_date_sk#139, d_year#140] + +(142) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#141, d_year#142] + +(143) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_shipto_date_sk#137] +Right keys [1]: [d_date_sk#141] +Join type: Inner +Join condition: None + +(144) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140, d_date_sk#141, d_year#142] + +(145) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#143, cd_marital_status#144] + +(146) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_cdemo_sk#109] +Right keys [1]: [cd_demo_sk#143] +Join type: Inner +Join condition: None + +(147) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_demo_sk#143, cd_marital_status#144] + +(148) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] + +(149) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_cdemo_sk#134] +Right keys [1]: [cd_demo_sk#145] +Join type: Inner +Join condition: NOT (cd_marital_status#144 = cd_marital_status#146) + +(150) Project [codegen id : 36] +Output [14]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144, cd_demo_sk#145, cd_marital_status#146] + +(151) ReusedExchange [Reuses operator id: 74] +Output [1]: [p_promo_sk#147] + +(152) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_promo_sk#113] +Right keys [1]: [p_promo_sk#147] +Join type: Inner +Join condition: None + +(153) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, p_promo_sk#147] + +(154) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#148, hd_income_band_sk#149] + +(155) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_hdemo_sk#110] +Right keys [1]: [hd_demo_sk#148] +Join type: Inner +Join condition: None + +(156) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_demo_sk#148, hd_income_band_sk#149] + +(157) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] + +(158) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_hdemo_sk#135] +Right keys [1]: [hd_demo_sk#150] +Join type: Inner +Join condition: None + +(159) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151] +Input [15]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_demo_sk#150, hd_income_band_sk#151] + +(160) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] + +(161) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_addr_sk#111] +Right keys [1]: [ca_address_sk#152] +Join type: Inner +Join condition: None + +(162) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] +Input [18]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] + +(163) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] + +(164) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_addr_sk#136] +Right keys [1]: [ca_address_sk#157] +Join type: Inner +Join condition: None + +(165) Project [codegen id : 36] +Output [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [21]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] + +(166) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#162] + +(167) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#149] +Right keys [1]: [ib_income_band_sk#162] +Join type: Inner +Join condition: None + +(168) Project [codegen id : 36] +Output [18]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [20]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#162] + +(169) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#163] + +(170) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#163] +Join type: Inner +Join condition: None + +(171) Project [codegen id : 36] +Output [17]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#163] + +(172) ReusedExchange [Reuses operator id: 109] +Output [2]: [i_item_sk#164, i_product_name#165] + +(173) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [i_item_sk#164] +Join type: Inner +Join condition: None + +(174) Project [codegen id : 36] +Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] + +(175) HashAggregate [codegen id : 36] +Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count#78, sum#166, sum#167, sum#168] +Results [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] + +(176) HashAggregate [codegen id : 36] +Input [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#114))#87, sum(UnscaledValue(ss_list_price#115))#88, sum(UnscaledValue(ss_coupon_amt#116))#89] +Results [8]: [i_item_sk#164 AS item_sk#172, s_store_name#131 AS store_name#173, s_zip#132 AS store_zip#174, d_year#129 AS syear#175, count(1)#86 AS cnt#176, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#87,17,2) AS s1#177, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#88,17,2) AS s2#178, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#89,17,2) AS s3#179] + +(177) CometColumnarExchange +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: hashpartitioning(item_sk#172, store_name#173, store_zip#174, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] + +(178) CometSort +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179], [item_sk#172 ASC NULLS FIRST, store_name#173 ASC NULLS FIRST, store_zip#174 ASC NULLS FIRST] + +(179) CometSortMergeJoin +Left output [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Right output [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#91, store_name#92, store_zip#93], [item_sk#172, store_name#173, store_zip#174], Inner, (cnt#176 <= cnt#103) + +(180) CometProject +Input [25]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + +(181) CometExchange +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: rangepartitioning(product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, s1#104 ASC NULLS FIRST, s1#177 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] + +(182) CometSort +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, s1#104 ASC NULLS FIRST, s1#177 ASC NULLS FIRST] + +(183) CometColumnarToRow [codegen id : 37] +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/extended.txt new file mode 100644 index 0000000000..0ca4716e97 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/extended.txt @@ -0,0 +1,271 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Project + : : : : +- BroadcastHashJoin + : : : : :- Project + : : : : : +- BroadcastHashJoin + : : : : : :- Project + : : : : : : +- BroadcastHashJoin + : : : : : : :- Project + : : : : : : : +- BroadcastHashJoin + : : : : : : : :- Project + : : : : : : : : +- BroadcastHashJoin + : : : : : : : : :- Project + : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : :- Project + : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : :- Project + : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : :- Project + : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : : : +- SortMergeJoin + : : : : : : : : : : : : : : : : :- CometColumnarToRow + : : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- Sort + : : : : : : : : : : : : : : : : +- Project + : : : : : : : : : : : : : : : : +- Filter + : : : : : : : : : : : : : : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_item_sk#1, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#2))#3,17,2) AS sale#4, sum(((cr_refunded_cash#5 + cr_reversed_charge#6) + cr_store_credit#7))#8 AS refund#9)] + : : : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : +- CometProject + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- BroadcastExchange + : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- BroadcastExchange + : : : : : : : : +- CometColumnarToRow + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : : +- BroadcastExchange + : : : : : : : +- CometColumnarToRow + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : : +- BroadcastExchange + : : : : : : +- CometColumnarToRow + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- BroadcastExchange + : : : : : +- CometColumnarToRow + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- BroadcastExchange + : : : : +- CometColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.income_band + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Project + : : : : +- BroadcastHashJoin + : : : : :- Project + : : : : : +- BroadcastHashJoin + : : : : : :- Project + : : : : : : +- BroadcastHashJoin + : : : : : : :- Project + : : : : : : : +- BroadcastHashJoin + : : : : : : : :- Project + : : : : : : : : +- BroadcastHashJoin + : : : : : : : : :- Project + : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : :- Project + : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : :- Project + : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : :- Project + : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : +- BroadcastHashJoin + : : : : : : : : : : : : : : :- Project + : : : : : : : : : : : : : : : +- SortMergeJoin + : : : : : : : : : : : : : : : :- CometColumnarToRow + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : +- Sort + : : : : : : : : : : : : : : : +- Project + : : : : : : : : : : : : : : : +- Filter + : : : : : : : : : : : : : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_item_sk#10, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#11))#3,17,2) AS sale#12, sum(((cr_refunded_cash#13 + cr_reversed_charge#14) + cr_store_credit#15))#8 AS refund#16)] + : : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- BroadcastExchange + : : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : +- BroadcastExchange + : : : : : : : : : +- CometColumnarToRow + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- BroadcastExchange + : : : : : : : : +- CometColumnarToRow + : : : : : : : : +- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : +- BroadcastExchange + : : : : : : : +- CometColumnarToRow + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : +- BroadcastExchange + : : : : : : +- CometColumnarToRow + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- BroadcastExchange + : : : : : +- CometColumnarToRow + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- BroadcastExchange + : : : : +- CometColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 128 out of 234 eligible operators (54%). Final plan contains 35 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/simplified.txt new file mode 100644 index 0000000000..89e96b849a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_datafusion/simplified.txt @@ -0,0 +1,251 @@ +WholeStageCodegen (37) + CometColumnarToRow + InputAdapter + CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometExchange [product_name,store_name,cnt,s1,s1] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #2 + WholeStageCodegen (18) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + InputAdapter + WholeStageCodegen (2) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (10) + CometColumnarToRow + InputAdapter + CometFilter [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + BroadcastExchange #14 + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (13) + CometColumnarToRow + InputAdapter + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + BroadcastExchange #16 + WholeStageCodegen (15) + CometColumnarToRow + InputAdapter + CometFilter [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + BroadcastExchange #17 + WholeStageCodegen (17) + CometColumnarToRow + InputAdapter + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #18 + WholeStageCodegen (36) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + InputAdapter + WholeStageCodegen (20) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + InputAdapter + BroadcastExchange #19 + WholeStageCodegen (21) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + InputAdapter + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + ReusedExchange [p_promo_sk] #13 + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/explain.txt new file mode 100644 index 0000000000..a998806186 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/explain.txt @@ -0,0 +1,412 @@ +== Physical Plan == +TakeOrderedAndProject (74) ++- * Filter (73) + +- Window (72) + +- WindowGroupLimit (71) + +- * CometColumnarToRow (70) + +- CometSort (69) + +- CometColumnarExchange (68) + +- WindowGroupLimit (67) + +- * CometColumnarToRow (66) + +- CometSort (65) + +- CometUnion (64) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + :- CometHashAggregate (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- CometHashAggregate (30) + : +- ReusedExchange (29) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometHashAggregate (35) + : +- ReusedExchange (34) + :- CometHashAggregate (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometHashAggregate (40) + : +- ReusedExchange (39) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometHashAggregate (45) + : +- ReusedExchange (44) + :- CometHashAggregate (53) + : +- CometExchange (52) + : +- CometHashAggregate (51) + : +- CometHashAggregate (50) + : +- ReusedExchange (49) + :- CometHashAggregate (58) + : +- CometExchange (57) + : +- CometHashAggregate (56) + : +- CometHashAggregate (55) + : +- ReusedExchange (54) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometHashAggregate (60) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] + +(6) CometBroadcastExchange +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] + +(9) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#11, s_store_id#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) + +(11) CometProject +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#12, 16)) AS s_store_id#13] + +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight + +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) + +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#15, 50)) AS i_brand#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#16, 50)) AS i_class#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#18, 50)) AS i_product_name#22] + +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] + +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight + +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] + +(21) CometHashAggregate +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] + +(22) CometExchange +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] + +(24) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32, sum#33, isEmpty#34] + +(25) CometHashAggregate +Input [10]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32, sum#33, isEmpty#34] +Keys [8]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32] +Functions [1]: [sum(coalesce((ss_sales_price#35 * cast(ss_quantity#36 as decimal(10,0))), 0.00))] + +(26) CometHashAggregate +Input [8]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sumsales#37] +Keys [7]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31] +Functions [1]: [partial_sum(sumsales#37)] + +(27) CometExchange +Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sum#38, isEmpty#39] +Arguments: hashpartitioning(i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometHashAggregate +Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sum#38, isEmpty#39] +Keys [7]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31] +Functions [1]: [sum(sumsales#37)] + +(29) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47, sum#48, isEmpty#49] + +(30) CometHashAggregate +Input [10]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47, sum#48, isEmpty#49] +Keys [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47] +Functions [1]: [sum(coalesce((ss_sales_price#50 * cast(ss_quantity#51 as decimal(10,0))), 0.00))] + +(31) CometHashAggregate +Input [7]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sumsales#52] +Keys [6]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45] +Functions [1]: [partial_sum(sumsales#52)] + +(32) CometExchange +Input [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sum#53, isEmpty#54] +Arguments: hashpartitioning(i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(33) CometHashAggregate +Input [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sum#53, isEmpty#54] +Keys [6]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45] +Functions [1]: [sum(sumsales#52)] + +(34) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62, sum#63, isEmpty#64] + +(35) CometHashAggregate +Input [10]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62, sum#63, isEmpty#64] +Keys [8]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62] +Functions [1]: [sum(coalesce((ss_sales_price#65 * cast(ss_quantity#66 as decimal(10,0))), 0.00))] + +(36) CometHashAggregate +Input [6]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sumsales#67] +Keys [5]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59] +Functions [1]: [partial_sum(sumsales#67)] + +(37) CometExchange +Input [7]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sum#68, isEmpty#69] +Arguments: hashpartitioning(i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(38) CometHashAggregate +Input [7]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sum#68, isEmpty#69] +Keys [5]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59] +Functions [1]: [sum(sumsales#67)] + +(39) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77, sum#78, isEmpty#79] + +(40) CometHashAggregate +Input [10]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77, sum#78, isEmpty#79] +Keys [8]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77] +Functions [1]: [sum(coalesce((ss_sales_price#80 * cast(ss_quantity#81 as decimal(10,0))), 0.00))] + +(41) CometHashAggregate +Input [5]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sumsales#82] +Keys [4]: [i_category#70, i_class#71, i_brand#72, i_product_name#73] +Functions [1]: [partial_sum(sumsales#82)] + +(42) CometExchange +Input [6]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sum#83, isEmpty#84] +Arguments: hashpartitioning(i_category#70, i_class#71, i_brand#72, i_product_name#73, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(43) CometHashAggregate +Input [6]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sum#83, isEmpty#84] +Keys [4]: [i_category#70, i_class#71, i_brand#72, i_product_name#73] +Functions [1]: [sum(sumsales#82)] + +(44) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92, sum#93, isEmpty#94] + +(45) CometHashAggregate +Input [10]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92, sum#93, isEmpty#94] +Keys [8]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92] +Functions [1]: [sum(coalesce((ss_sales_price#95 * cast(ss_quantity#96 as decimal(10,0))), 0.00))] + +(46) CometHashAggregate +Input [4]: [i_category#85, i_class#86, i_brand#87, sumsales#97] +Keys [3]: [i_category#85, i_class#86, i_brand#87] +Functions [1]: [partial_sum(sumsales#97)] + +(47) CometExchange +Input [5]: [i_category#85, i_class#86, i_brand#87, sum#98, isEmpty#99] +Arguments: hashpartitioning(i_category#85, i_class#86, i_brand#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(48) CometHashAggregate +Input [5]: [i_category#85, i_class#86, i_brand#87, sum#98, isEmpty#99] +Keys [3]: [i_category#85, i_class#86, i_brand#87] +Functions [1]: [sum(sumsales#97)] + +(49) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107, sum#108, isEmpty#109] + +(50) CometHashAggregate +Input [10]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107, sum#108, isEmpty#109] +Keys [8]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107] +Functions [1]: [sum(coalesce((ss_sales_price#110 * cast(ss_quantity#111 as decimal(10,0))), 0.00))] + +(51) CometHashAggregate +Input [3]: [i_category#100, i_class#101, sumsales#112] +Keys [2]: [i_category#100, i_class#101] +Functions [1]: [partial_sum(sumsales#112)] + +(52) CometExchange +Input [4]: [i_category#100, i_class#101, sum#113, isEmpty#114] +Arguments: hashpartitioning(i_category#100, i_class#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(53) CometHashAggregate +Input [4]: [i_category#100, i_class#101, sum#113, isEmpty#114] +Keys [2]: [i_category#100, i_class#101] +Functions [1]: [sum(sumsales#112)] + +(54) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122, sum#123, isEmpty#124] + +(55) CometHashAggregate +Input [10]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122, sum#123, isEmpty#124] +Keys [8]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122] +Functions [1]: [sum(coalesce((ss_sales_price#125 * cast(ss_quantity#126 as decimal(10,0))), 0.00))] + +(56) CometHashAggregate +Input [2]: [i_category#115, sumsales#127] +Keys [1]: [i_category#115] +Functions [1]: [partial_sum(sumsales#127)] + +(57) CometExchange +Input [3]: [i_category#115, sum#128, isEmpty#129] +Arguments: hashpartitioning(i_category#115, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(58) CometHashAggregate +Input [3]: [i_category#115, sum#128, isEmpty#129] +Keys [1]: [i_category#115] +Functions [1]: [sum(sumsales#127)] + +(59) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137, sum#138, isEmpty#139] + +(60) CometHashAggregate +Input [10]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137, sum#138, isEmpty#139] +Keys [8]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137] +Functions [1]: [sum(coalesce((ss_sales_price#140 * cast(ss_quantity#141 as decimal(10,0))), 0.00))] + +(61) CometHashAggregate +Input [1]: [sumsales#142] +Keys: [] +Functions [1]: [partial_sum(sumsales#142)] + +(62) CometExchange +Input [2]: [sum#143, isEmpty#144] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(63) CometHashAggregate +Input [2]: [sum#143, isEmpty#144] +Keys: [] +Functions [1]: [sum(sumsales#142)] + +(64) CometUnion +Child 0 Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Child 1 Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#154, sumsales#155] +Child 2 Input [9]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#156, s_store_id#157, sumsales#158] +Child 3 Input [9]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#159, d_moy#160, s_store_id#161, sumsales#162] +Child 4 Input [9]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#163, d_qoy#164, d_moy#165, s_store_id#166, sumsales#167] +Child 5 Input [9]: [i_category#85, i_class#86, i_brand#87, i_product_name#168, d_year#169, d_qoy#170, d_moy#171, s_store_id#172, sumsales#173] +Child 6 Input [9]: [i_category#100, i_class#101, i_brand#174, i_product_name#175, d_year#176, d_qoy#177, d_moy#178, s_store_id#179, sumsales#180] +Child 7 Input [9]: [i_category#115, i_class#181, i_brand#182, i_product_name#183, d_year#184, d_qoy#185, d_moy#186, s_store_id#187, sumsales#188] +Child 8 Input [9]: [i_category#189, i_class#190, i_brand#191, i_product_name#192, d_year#193, d_qoy#194, d_moy#195, s_store_id#196, sumsales#197] + +(65) CometSort +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153], [i_category#145 ASC NULLS FIRST, sumsales#153 DESC NULLS LAST] + +(66) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] + +(67) WindowGroupLimit +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145], [sumsales#153 DESC NULLS LAST], rank(sumsales#153), 100, Partial + +(68) CometColumnarExchange +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: hashpartitioning(i_category#145, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(69) CometSort +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153], [i_category#145 ASC NULLS FIRST, sumsales#153 DESC NULLS LAST] + +(70) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] + +(71) WindowGroupLimit +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145], [sumsales#153 DESC NULLS LAST], rank(sumsales#153), 100, Final + +(72) Window +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [rank(sumsales#153) windowspecdefinition(i_category#145, sumsales#153 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#198], [i_category#145], [sumsales#153 DESC NULLS LAST] + +(73) Filter [codegen id : 3] +Input [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] +Condition : (rk#198 <= 100) + +(74) TakeOrderedAndProject +Input [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] +Arguments: 100, [i_category#145 ASC NULLS FIRST, i_class#146 ASC NULLS FIRST, i_brand#147 ASC NULLS FIRST, i_product_name#148 ASC NULLS FIRST, d_year#149 ASC NULLS FIRST, d_qoy#150 ASC NULLS FIRST, d_moy#151 ASC NULLS FIRST, s_store_id#152 ASC NULLS FIRST, sumsales#153 ASC NULLS FIRST, rk#198 ASC NULLS FIRST], [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/extended.txt new file mode 100644 index 0000000000..b9c702d0ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/extended.txt @@ -0,0 +1,244 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- CometColumnarToRow + +- CometSort + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 235 out of 240 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..45089e6274 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_datafusion/simplified.txt @@ -0,0 +1,80 @@ +TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] + WholeStageCodegen (3) + Filter [rk] + InputAdapter + Window [sumsales,i_category] + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] + CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] + CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] + CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name] #9 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] + CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand] #10 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] + CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class] #11 + CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] + CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category] #12 + CometHashAggregate [sumsales] [i_category,sum,isEmpty] + CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange #13 + CometHashAggregate [sumsales] [sum,isEmpty] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/explain.txt new file mode 100644 index 0000000000..7ef9ea93c7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/explain.txt @@ -0,0 +1,375 @@ +== Physical Plan == +TakeOrderedAndProject (66) ++- * Project (65) + +- Window (64) + +- * CometColumnarToRow (63) + +- CometSort (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometColumnarExchange (59) + +- * HashAggregate (58) + +- Union (57) + :- * HashAggregate (42) + : +- * CometColumnarToRow (41) + : +- CometColumnarExchange (40) + : +- * HashAggregate (39) + : +- * Project (38) + : +- * BroadcastHashJoin Inner BuildRight (37) + : :- * CometColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (36) + : +- * Project (35) + : +- * BroadcastHashJoin LeftSemi BuildRight (34) + : :- * CometColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometNativeScan parquet spark_catalog.default.store (10) + : +- BroadcastExchange (33) + : +- * Project (32) + : +- * Filter (31) + : +- Window (30) + : +- WindowGroupLimit (29) + : +- * Sort (28) + : +- * HashAggregate (27) + : +- * CometColumnarToRow (26) + : +- CometExchange (25) + : +- CometHashAggregate (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.store (15) + : +- ReusedExchange (21) + :- * HashAggregate (49) + : +- * CometColumnarToRow (48) + : +- CometColumnarExchange (47) + : +- * HashAggregate (46) + : +- * HashAggregate (45) + : +- * CometColumnarToRow (44) + : +- ReusedExchange (43) + +- * HashAggregate (56) + +- * CometColumnarToRow (55) + +- CometColumnarExchange (54) + +- * HashAggregate (53) + +- * HashAggregate (52) + +- * CometColumnarToRow (51) + +- ReusedExchange (50) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] + +(9) CometColumnarToRow [codegen id : 4] +Input [2]: [ss_store_sk#1, ss_net_profit#2] + +(10) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#6, s_county#7, s_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) + +(12) CometColumnarToRow [codegen id : 3] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] + +(13) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) + +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#12, s_state#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) + +(17) CometProject +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#14] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight + +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] + +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight + +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] + +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] + +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(26) CometColumnarToRow [codegen id : 1] +Input [2]: [s_state#14, sum#16] + +(27) HashAggregate [codegen id : 1] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] +Results [3]: [s_state#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#14] + +(28) Sort [codegen id : 1] +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 + +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final + +(30) Window +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [rank(_w0#18) windowspecdefinition(s_state#14, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#14], [_w0#18 DESC NULLS LAST] + +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] +Condition : (ranking#19 <= 5) + +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] + +(33) BroadcastExchange +Input [1]: [s_state#14] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] + +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2))] +Right keys [1]: [s_state#14] +Join type: LeftSemi +Join condition: None + +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2)) AS s_state#20] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] + +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [ss_store_sk#1] +Right keys [1]: [s_store_sk#6] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_county#7, s_state#20] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#20] + +(39) HashAggregate [codegen id : 4] +Input [3]: [ss_net_profit#2, s_county#7, s_state#20] +Keys [2]: [s_state#20, s_county#7] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] +Aggregate Attributes [1]: [sum#21] +Results [3]: [s_state#20, s_county#7, sum#22] + +(40) CometColumnarExchange +Input [3]: [s_state#20, s_county#7, sum#22] +Arguments: hashpartitioning(s_state#20, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(41) CometColumnarToRow [codegen id : 5] +Input [3]: [s_state#20, s_county#7, sum#22] + +(42) HashAggregate [codegen id : 5] +Input [3]: [s_state#20, s_county#7, sum#22] +Keys [2]: [s_state#20, s_county#7] +Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) as decimal(27,2)) AS total_sum#24, s_state#20 AS s_state#25, s_county#7 AS s_county#26, 0 AS g_state#27, 0 AS g_county#28, 0 AS lochierarchy#29] + +(43) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#30, s_county#31, sum#32] + +(44) CometColumnarToRow [codegen id : 10] +Input [3]: [s_state#30, s_county#31, sum#32] + +(45) HashAggregate [codegen id : 10] +Input [3]: [s_state#30, s_county#31, sum#32] +Keys [2]: [s_state#30, s_county#31] +Functions [1]: [sum(UnscaledValue(ss_net_profit#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#33))#23] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#33))#23,17,2) AS total_sum#34, s_state#30] + +(46) HashAggregate [codegen id : 10] +Input [2]: [total_sum#34, s_state#30] +Keys [1]: [s_state#30] +Functions [1]: [partial_sum(total_sum#34)] +Aggregate Attributes [2]: [sum#35, isEmpty#36] +Results [3]: [s_state#30, sum#37, isEmpty#38] + +(47) CometColumnarExchange +Input [3]: [s_state#30, sum#37, isEmpty#38] +Arguments: hashpartitioning(s_state#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(48) CometColumnarToRow [codegen id : 11] +Input [3]: [s_state#30, sum#37, isEmpty#38] + +(49) HashAggregate [codegen id : 11] +Input [3]: [s_state#30, sum#37, isEmpty#38] +Keys [1]: [s_state#30] +Functions [1]: [sum(total_sum#34)] +Aggregate Attributes [1]: [sum(total_sum#34)#39] +Results [6]: [sum(total_sum#34)#39 AS total_sum#40, s_state#30, null AS s_county#41, 0 AS g_state#42, 1 AS g_county#43, 1 AS lochierarchy#44] + +(50) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#45, s_county#46, sum#47] + +(51) CometColumnarToRow [codegen id : 16] +Input [3]: [s_state#45, s_county#46, sum#47] + +(52) HashAggregate [codegen id : 16] +Input [3]: [s_state#45, s_county#46, sum#47] +Keys [2]: [s_state#45, s_county#46] +Functions [1]: [sum(UnscaledValue(ss_net_profit#48))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#48))#23] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#48))#23,17,2) AS total_sum#49] + +(53) HashAggregate [codegen id : 16] +Input [1]: [total_sum#49] +Keys: [] +Functions [1]: [partial_sum(total_sum#49)] +Aggregate Attributes [2]: [sum#50, isEmpty#51] +Results [2]: [sum#52, isEmpty#53] + +(54) CometColumnarExchange +Input [2]: [sum#52, isEmpty#53] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(55) CometColumnarToRow [codegen id : 17] +Input [2]: [sum#52, isEmpty#53] + +(56) HashAggregate [codegen id : 17] +Input [2]: [sum#52, isEmpty#53] +Keys: [] +Functions [1]: [sum(total_sum#49)] +Aggregate Attributes [1]: [sum(total_sum#49)#54] +Results [6]: [sum(total_sum#49)#54 AS total_sum#55, null AS s_state#56, null AS s_county#57, 1 AS g_state#58, 1 AS g_county#59, 2 AS lochierarchy#60] + +(57) Union + +(58) HashAggregate [codegen id : 18] +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Keys [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Functions: [] +Aggregate Attributes: [] +Results [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] + +(59) CometColumnarExchange +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Arguments: hashpartitioning(total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(60) CometHashAggregate +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Keys [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Functions: [] + +(61) CometExchange +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: hashpartitioning(lochierarchy#29, _w0#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(62) CometSort +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61], [lochierarchy#29 ASC NULLS FIRST, _w0#61 ASC NULLS FIRST, total_sum#24 DESC NULLS LAST] + +(63) CometColumnarToRow [codegen id : 19] +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] + +(64) Window +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: [rank(total_sum#24) windowspecdefinition(lochierarchy#29, _w0#61, total_sum#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#62], [lochierarchy#29, _w0#61], [total_sum#24 DESC NULLS LAST] + +(65) Project [codegen id : 20] +Output [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] +Input [6]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61, rank_within_parent#62] + +(66) TakeOrderedAndProject +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] +Arguments: 100, [lochierarchy#29 DESC NULLS LAST, CASE WHEN (lochierarchy#29 = 0) THEN s_state#25 END ASC NULLS FIRST, rank_within_parent#62 ASC NULLS FIRST], [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/extended.txt new file mode 100644 index 0000000000..6f68bf7751 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/extended.txt @@ -0,0 +1,155 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Filter + : +- Window + : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : +- Sort + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_state#1, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#3,17,2) AS _w0#4, s_state#1)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Filter + : +- Window + : +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + : +- Sort + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_state#1, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#3,17,2) AS _w0#4, s_state#1)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Filter + +- Window + +- WindowGroupLimit [COMET: WindowGroupLimit is not supported] + +- Sort + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_state#1, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#3,17,2) AS _w0#4, s_state#1)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 87 out of 138 eligible operators (63%). Final plan contains 15 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..52138d1138 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_datafusion/simplified.txt @@ -0,0 +1,92 @@ +TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] + WholeStageCodegen (20) + Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] + InputAdapter + Window [total_sum,lochierarchy,_w0] + WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometSort [total_sum,s_state,s_county,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_state] [total_sum,s_state,s_county,lochierarchy,_w0,g_county] + CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 + WholeStageCodegen (18) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county] #3 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] + Project [ss_net_profit,s_county,s_state] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + Sort [s_state,_w0] + HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_state] #7 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #8 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (11) + HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state] #9 + WholeStageCodegen (10) + HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (16) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/explain.txt new file mode 100644 index 0000000000..e7aa80f0d2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/explain.txt @@ -0,0 +1,369 @@ +== Physical Plan == +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) + +(3) CometNativeScan parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(inv_date_sk#12)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) + +(5) CometBroadcastExchange +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] + +(6) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight + +(7) CometProject +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] + +(8) CometNativeScan parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) + +(10) CometBroadcastExchange +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] + +(11) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight + +(12) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] + +(13) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_desc#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) + +(15) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] + +(16) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight + +(17) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(18) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#17, cd_marital_status#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((isnotnull(cd_marital_status#18) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#18, 1)) = M)) AND isnotnull(cd_demo_sk#17)) + +(20) CometProject +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] + +(21) CometBroadcastExchange +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] + +(22) CometBroadcastHashJoin +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight + +(23) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(24) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((isnotnull(hd_buy_potential#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#20, 15)) = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) + +(26) CometProject +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] + +(27) CometBroadcastExchange +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] + +(28) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight + +(29) CometProject +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(30) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) + +(32) CometProject +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] + +(33) CometBroadcastExchange +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] + +(34) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight + +(35) CometProject +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] + +(36) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_week_seq#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct + +(37) CometFilter +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) + +(38) CometBroadcastExchange +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] + +(39) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight + +(40) CometProject +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] + +(41) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_date#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] +ReadSchema: struct + +(42) CometFilter +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) + +(43) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] + +(44) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight + +(45) CometProject +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(46) CometNativeScan parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(47) CometFilter +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) + +(48) CometBroadcastExchange +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] + +(49) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight + +(50) CometProject +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(51) CometExchange +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(52) CometSort +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] + +(53) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(54) CometFilter +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) + +(55) CometProject +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] + +(56) CometExchange +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(57) CometSort +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter + +(59) CometProject +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(60) CometHashAggregate +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] +Functions [1]: [partial_count(1)] + +(61) CometExchange +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(62) CometHashAggregate +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] +Functions [1]: [count(1)] + +(63) CometTakeOrderedAndProject +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + +(64) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/extended.txt new file mode 100644 index 0000000000..3bf43ef8c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/extended.txt @@ -0,0 +1,66 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_returns + +Comet accelerated 63 out of 63 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/simplified.txt new file mode 100644 index 0000000000..073bd57a8c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_datafusion/simplified.txt @@ -0,0 +1,66 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] + CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] + CometProject [w_warehouse_name,i_item_desc,d_week_seq] + CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometExchange [cs_item_sk,cs_order_number] #2 + CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 + CometFilter [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 + CometFilter [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/explain.txt new file mode 100644 index 0000000000..7b07beadf9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/explain.txt @@ -0,0 +1,406 @@ +== Physical Plan == +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometNativeScan parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) + + +(1) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#7] + +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) + +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] + +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight + +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] + +(17) HashAggregate [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#14] +Results [2]: [c_customer_id#5 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#14,17,2) AS year_total#16] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#15, year_total#16] +Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) + +(19) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)))) + +(21) CometProject +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Arguments: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)) AS c_customer_id#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#23] + +(22) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] + +(23) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23] +Right output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_sk#17], [ss_customer_sk#24], Inner, BuildRight + +(24) CometProject +Input [7]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23, ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] + +(25) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_year#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#27, d_year#28] +Condition : (((isnotnull(d_year#28) AND (d_year#28 = 2002)) AND d_year#28 IN (2001,2002)) AND isnotnull(d_date_sk#27)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_year#28] +Arguments: [d_date_sk#27, d_year#28] + +(28) CometBroadcastHashJoin +Left output [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] +Right output [2]: [d_date_sk#27, d_year#28] +Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight + +(29) CometProject +Input [7]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] + +(30) CometHashAggregate +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#25))] + +(31) CometExchange +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] + +(33) HashAggregate [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [sum(UnscaledValue(ss_net_paid#25))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#25))#14] +Results [4]: [c_customer_id#21 AS customer_id#30, c_first_name#22 AS customer_first_name#31, c_last_name#23 AS customer_last_name#32, MakeDecimal(sum(UnscaledValue(ss_net_paid#25))#14,17,2) AS year_total#33] + +(34) BroadcastExchange +Input [4]: [customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#30] +Join type: Inner +Join condition: None + +(36) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(37) CometFilter +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Condition : (isnotnull(c_customer_sk#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)))) + +(38) CometProject +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Arguments: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40], [c_customer_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)) AS c_customer_id#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#36, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#37, 30)) AS c_last_name#40] + +(39) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_customer_sk#41) + +(41) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] + +(42) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40] +Right output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_sk#34], [ws_bill_customer_sk#41], Inner, BuildRight + +(43) CometProject +Input [7]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40, ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] + +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#44, d_year#45] + +(45) CometBroadcastHashJoin +Left output [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] +Right output [2]: [d_date_sk#44, d_year#45] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(46) CometProject +Input [7]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] + +(47) CometHashAggregate +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#42))] + +(48) CometExchange +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Arguments: hashpartitioning(c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(49) CometColumnarToRow [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] + +(50) HashAggregate [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_net_paid#42))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#42))#47] +Results [2]: [c_customer_id#38 AS customer_id#48, MakeDecimal(sum(UnscaledValue(ws_net_paid#42))#47,17,2) AS year_total#49] + +(51) Filter [codegen id : 2] +Input [2]: [customer_id#48, year_total#49] +Condition : (isnotnull(year_total#49) AND (year_total#49 > 0.00)) + +(52) BroadcastExchange +Input [2]: [customer_id#48, year_total#49] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#48] +Join type: Inner +Join condition: None + +(54) Project [codegen id : 4] +Output [7]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49] +Input [8]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, customer_id#48, year_total#49] + +(55) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(56) CometFilter +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Condition : (isnotnull(c_customer_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)))) + +(57) CometProject +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Arguments: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56], [c_customer_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)) AS c_customer_id#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#52, 20)) AS c_first_name#55, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#53, 30)) AS c_last_name#56] + +(58) ReusedExchange [Reuses operator id: 41] +Output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] + +(59) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56] +Right output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#50], [ws_bill_customer_sk#57], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56, ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] + +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#60, d_year#61] + +(62) CometBroadcastHashJoin +Left output [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(63) CometProject +Input [7]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] + +(64) CometHashAggregate +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#58))] + +(65) CometExchange +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] + +(67) HashAggregate [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [sum(UnscaledValue(ws_net_paid#58))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#58))#47] +Results [2]: [c_customer_id#54 AS customer_id#63, MakeDecimal(sum(UnscaledValue(ws_net_paid#58))#47,17,2) AS year_total#64] + +(68) BroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#63] +Join type: Inner +Join condition: (CASE WHEN (year_total#49 > 0.00) THEN (year_total#64 / year_total#49) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#33 / year_total#16) END) + +(70) Project [codegen id : 4] +Output [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Input [9]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49, customer_id#63, year_total#64] + +(71) TakeOrderedAndProject +Input [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Arguments: 100, [customer_first_name#31 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST, customer_last_name#32 ASC NULLS FIRST], [customer_id#30, customer_first_name#31, customer_last_name#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/extended.txt new file mode 100644 index 0000000000..696be0ba76 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/extended.txt @@ -0,0 +1,81 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- BroadcastHashJoin + : : :- Filter + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#1 AS customer_id#2, MakeDecimal(sum(UnscaledValue(ss_net_paid#3))#4,17,2) AS year_total#5)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#6 AS customer_id#7, c_first_name#8 AS customer_first_name#9, c_last_name#10 AS customer_last_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#12))#4,17,2) AS year_total#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(c_customer_id#14 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ws_net_paid#16))#17,17,2) AS year_total#18)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: Vector(c_customer_id#19 AS customer_id#20, MakeDecimal(sum(UnscaledValue(ws_net_paid#21))#17,17,2) AS year_total#22)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 60 out of 75 eligible operators (80%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/simplified.txt new file mode 100644 index 0000000000..da3e749ab4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_datafusion/simplified.txt @@ -0,0 +1,82 @@ +TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] + WholeStageCodegen (4) + Project [customer_id,customer_first_name,customer_last_name] + BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] + Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] + BroadcastHashJoin [customer_id,customer_id] + BroadcastHashJoin [customer_id,customer_id] + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/explain.txt new file mode 100644 index 0000000000..a4b1ef8608 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/explain.txt @@ -0,0 +1,693 @@ +== Physical Plan == +* CometColumnarToRow (126) ++- CometTakeOrderedAndProject (125) + +- CometProject (124) + +- CometSortMergeJoin (123) + :- CometSort (67) + : +- CometColumnarExchange (66) + : +- * Filter (65) + : +- * HashAggregate (64) + : +- * CometColumnarToRow (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) + +- CometSort (122) + +- CometColumnarExchange (121) + +- * Filter (120) + +- * HashAggregate (119) + +- * CometColumnarToRow (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometHashAggregate (115) + +- CometExchange (114) + +- CometHashAggregate (113) + +- CometUnion (112) + :- CometProject (83) + : +- CometSortMergeJoin (82) + : :- CometSort (79) + : : +- CometExchange (78) + : : +- CometProject (77) + : : +- CometBroadcastHashJoin (76) + : : :- CometProject (72) + : : : +- CometBroadcastHashJoin (71) + : : : :- CometFilter (69) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (68) + : : : +- ReusedExchange (70) + : : +- CometBroadcastExchange (75) + : : +- CometFilter (74) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (73) + : +- CometSort (81) + : +- ReusedExchange (80) + :- CometProject (97) + : +- CometSortMergeJoin (96) + : :- CometSort (93) + : : +- CometExchange (92) + : : +- CometProject (91) + : : +- CometBroadcastHashJoin (90) + : : :- CometProject (88) + : : : +- CometBroadcastHashJoin (87) + : : : :- CometFilter (85) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (84) + : : : +- ReusedExchange (86) + : : +- ReusedExchange (89) + : +- CometSort (95) + : +- ReusedExchange (94) + +- CometProject (111) + +- CometSortMergeJoin (110) + :- CometSort (107) + : +- CometExchange (106) + : +- CometProject (105) + : +- CometBroadcastHashJoin (104) + : :- CometProject (102) + : : +- CometBroadcastHashJoin (101) + : : :- CometFilter (99) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (98) + : : +- ReusedExchange (100) + : +- ReusedExchange (103) + +- CometSort (109) + +- ReusedExchange (108) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Condition : isnotnull(cs_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((isnotnull(i_category#10) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) + +(5) CometProject +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(6) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(7) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#12, d_year#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] + +(12) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight + +(13) CometProject +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] + +(14) CometExchange +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(15) CometSort +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] + +(16) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(17) CometFilter +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) + +(18) CometProject +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] + +(19) CometExchange +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] + +(21) CometSortMergeJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter + +(22) CometProject +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] + +(23) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(24) CometFilter +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) + +(25) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] + +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight + +(27) CometProject +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] + +(28) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#31, d_year#32] + +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight + +(30) CometProject +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] + +(31) CometExchange +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(32) CometSort +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] + +(33) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) + +(35) CometProject +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] + +(36) CometExchange +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(37) CometSort +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] + +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter + +(39) CometProject +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] + +(40) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(41) CometFilter +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) + +(42) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight + +(44) CometProject +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#50, d_year#51] + +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight + +(47) CometProject +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] + +(48) CometExchange +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(49) CometSort +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] + +(50) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) + +(52) CometProject +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] + +(53) CometExchange +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(54) CometSort +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter + +(56) CometProject +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] + +(57) CometUnion +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] + +(58) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Functions: [] + +(59) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(60) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Functions: [] + +(61) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] + +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(63) CometColumnarToRow [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] + +(64) HashAggregate [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] +Aggregate Attributes [2]: [sum(sales_cnt#19)#61, sum(UnscaledValue(sales_amt#20))#62] +Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#61 AS sales_cnt#63, MakeDecimal(sum(UnscaledValue(sales_amt#20))#62,18,2) AS sales_amt#64] + +(65) Filter [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Condition : isnotnull(sales_cnt#63) + +(66) CometColumnarExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(67) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] + +(68) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#69)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(69) CometFilter +Input [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Condition : isnotnull(cs_item_sk#65) + +(70) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(71) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Right output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65], [i_item_sk#70], Inner, BuildRight + +(72) CometProject +Input [10]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(73) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#75, d_year#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(74) CometFilter +Input [2]: [d_date_sk#75, d_year#76] +Condition : ((isnotnull(d_year#76) AND (d_year#76 = 2001)) AND isnotnull(d_date_sk#75)) + +(75) CometBroadcastExchange +Input [2]: [d_date_sk#75, d_year#76] +Arguments: [d_date_sk#75, d_year#76] + +(76) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Right output [2]: [d_date_sk#75, d_year#76] +Arguments: [cs_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight + +(77) CometProject +Input [11]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_date_sk#75, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] + +(78) CometExchange +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: hashpartitioning(cs_order_number#66, cs_item_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(79) CometSort +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_order_number#66 ASC NULLS FIRST, cs_item_sk#65 ASC NULLS FIRST] + +(80) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] + +(81) CometSort +Input [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80], [cr_order_number#78 ASC NULLS FIRST, cr_item_sk#77 ASC NULLS FIRST] + +(82) CometSortMergeJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Right output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cs_order_number#66, cs_item_sk#65], [cr_order_number#78, cr_item_sk#77], LeftOuter + +(83) CometProject +Input [13]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76, cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82], [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, (cs_quantity#67 - coalesce(cr_return_quantity#79, 0)) AS sales_cnt#81, (cs_ext_sales_price#68 - coalesce(cr_return_amount#80, 0.00)) AS sales_amt#82] + +(84) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(85) CometFilter +Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Condition : isnotnull(ss_item_sk#83) + +(86) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(87) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Right output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83], [i_item_sk#88], Inner, BuildRight + +(88) CometProject +Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(89) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#93, d_year#94] + +(90) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ss_sold_date_sk#87], [d_date_sk#93], Inner, BuildRight + +(91) CometProject +Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] + +(92) CometExchange +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(93) CometSort +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] + +(94) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] + +(95) CometSort +Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98], [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST] + +(96) CometSortMergeJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Right output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#96, sr_item_sk#95], LeftOuter + +(97) CometProject +Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100], [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#99, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#100] + +(98) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#105)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(99) CometFilter +Input [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Condition : isnotnull(ws_item_sk#101) + +(100) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(101) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101], [i_item_sk#106], Inner, BuildRight + +(102) CometProject +Input [10]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(103) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#111, d_year#112] + +(104) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Right output [2]: [d_date_sk#111, d_year#112] +Arguments: [ws_sold_date_sk#105], [d_date_sk#111], Inner, BuildRight + +(105) CometProject +Input [11]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] + +(106) CometExchange +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: hashpartitioning(ws_order_number#102, ws_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(107) CometSort +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#102 ASC NULLS FIRST, ws_item_sk#101 ASC NULLS FIRST] + +(108) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] + +(109) CometSort +Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] + +(110) CometSortMergeJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [ws_order_number#102, ws_item_sk#101], [wr_order_number#114, wr_item_sk#113], LeftOuter + +(111) CometProject +Input [13]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#103 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#117, (ws_ext_sales_price#104 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#118] + +(112) CometUnion +Child 0 Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Child 1 Input [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100] +Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118] + +(113) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] + +(114) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] + +(115) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] + +(116) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [partial_sum(sales_cnt#81), partial_sum(UnscaledValue(sales_amt#82))] + +(117) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] + +(118) CometColumnarToRow [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] + +(119) HashAggregate [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [sum(sales_cnt#81), sum(UnscaledValue(sales_amt#82))] +Aggregate Attributes [2]: [sum(sales_cnt#81)#61, sum(UnscaledValue(sales_amt#82))#62] +Results [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum(sales_cnt#81)#61 AS sales_cnt#121, MakeDecimal(sum(UnscaledValue(sales_amt#82))#62,18,2) AS sales_amt#122] + +(120) Filter [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Condition : isnotnull(sales_cnt#121) + +(121) CometColumnarExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: hashpartitioning(i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] + +(122) CometSort +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122], [i_brand_id#71 ASC NULLS FIRST, i_class_id#72 ASC NULLS FIRST, i_category_id#73 ASC NULLS FIRST, i_manufact_id#74 ASC NULLS FIRST] + +(123) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Right output [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], Inner, ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#121 as decimal(17,2))) < 0.90000000000000000000) + +(124) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], [d_year#76 AS prev_year#123, d_year#13 AS year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#121 AS prev_yr_cnt#125, sales_cnt#63 AS curr_yr_cnt#126, (sales_cnt#63 - sales_cnt#121) AS sales_cnt_diff#127, (sales_amt#64 - sales_amt#122) AS sales_amt_diff#128] + +(125) CometTakeOrderedAndProject +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#127 ASC NULLS FIRST,sales_amt_diff#128 ASC NULLS FIRST], output=[prev_year#123,year#124,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#125,curr_yr_cnt#126,sales_cnt_diff#127,sales_amt_diff#128]), [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], 100, 0, [sales_cnt_diff#127 ASC NULLS FIRST, sales_amt_diff#128 ASC NULLS FIRST], [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] + +(126) CometColumnarToRow [codegen id : 3] +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/extended.txt new file mode 100644 index 0000000000..dc2ec3af0b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/extended.txt @@ -0,0 +1,160 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#1, i_brand_id#2, i_class_id#3, i_category_id#4, i_manufact_id#5, sum(sales_cnt#6)#7 AS sales_cnt#8, MakeDecimal(sum(UnscaledValue(sales_amt#9))#10,18,2) AS sales_amt#11)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_returns + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_returns + +- CometSort + +- CometColumnarExchange + +- Filter + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#12, i_brand_id#13, i_class_id#14, i_category_id#15, i_manufact_id#16, sum(sales_cnt#17)#7 AS sales_cnt#18, MakeDecimal(sum(UnscaledValue(sales_amt#19))#10,18,2) AS sales_amt#20)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_returns + +Comet accelerated 151 out of 155 eligible operators (97%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/simplified.txt new file mode 100644 index 0000000000..557bee8d99 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_datafusion/simplified.txt @@ -0,0 +1,132 @@ +WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + WholeStageCodegen (1) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + WholeStageCodegen (2) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/explain.txt new file mode 100644 index 0000000000..196d3262af --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/explain.txt @@ -0,0 +1,600 @@ +== Physical Plan == +* CometColumnarToRow (105) ++- CometTakeOrderedAndProject (104) + +- CometHashAggregate (103) + +- CometColumnarExchange (102) + +- * HashAggregate (101) + +- Union (100) + :- * HashAggregate (85) + : +- * CometColumnarToRow (84) + : +- CometColumnarExchange (83) + : +- * HashAggregate (82) + : +- Union (81) + : :- * Project (32) + : : +- * BroadcastHashJoin LeftOuter BuildRight (31) + : : :- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.store (9) + : : +- BroadcastExchange (30) + : : +- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (18) + : : : +- ReusedExchange (20) + : : +- ReusedExchange (23) + : :- * Project (51) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (50) + : : :- BroadcastExchange (41) + : : : +- * HashAggregate (40) + : : : +- * CometColumnarToRow (39) + : : : +- CometExchange (38) + : : : +- CometHashAggregate (37) + : : : +- CometProject (36) + : : : +- CometBroadcastHashJoin (35) + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (33) + : : : +- ReusedExchange (34) + : : +- * HashAggregate (49) + : : +- * CometColumnarToRow (48) + : : +- CometExchange (47) + : : +- CometHashAggregate (46) + : : +- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns (42) + : : +- ReusedExchange (43) + : +- * Project (80) + : +- * BroadcastHashJoin LeftOuter BuildRight (79) + : :- * HashAggregate (65) + : : +- * CometColumnarToRow (64) + : : +- CometExchange (63) + : : +- CometHashAggregate (62) + : : +- CometProject (61) + : : +- CometBroadcastHashJoin (60) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometFilter (53) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (52) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (59) + : : +- CometFilter (58) + : : +- CometNativeScan parquet spark_catalog.default.web_page (57) + : +- BroadcastExchange (78) + : +- * HashAggregate (77) + : +- * CometColumnarToRow (76) + : +- CometExchange (75) + : +- CometHashAggregate (74) + : +- CometProject (73) + : +- CometBroadcastHashJoin (72) + : :- CometProject (70) + : : +- CometBroadcastHashJoin (69) + : : :- CometFilter (67) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (66) + : : +- ReusedExchange (68) + : +- ReusedExchange (71) + :- * HashAggregate (92) + : +- * CometColumnarToRow (91) + : +- CometColumnarExchange (90) + : +- * HashAggregate (89) + : +- * HashAggregate (88) + : +- * CometColumnarToRow (87) + : +- ReusedExchange (86) + +- * HashAggregate (99) + +- * CometColumnarToRow (98) + +- CometColumnarExchange (97) + +- * HashAggregate (96) + +- * HashAggregate (95) + +- * CometColumnarToRow (94) + +- ReusedExchange (93) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_date#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] + +(9) CometNativeScan parquet spark_catalog.default.store +Output [1]: [s_store_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) + +(11) CometBroadcastExchange +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] + +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight + +(13) CometProject +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] + +(14) CometHashAggregate +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] +Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] + +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] + +(17) HashAggregate [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] +Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#10, sum(UnscaledValue(ss_net_profit#3))#11] +Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS sales#12, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#11,17,2) AS profit#13] + +(18) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#17)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(19) CometFilter +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Condition : isnotnull(sr_store_sk#14) + +(20) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] + +(21) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight + +(22) CometProject +Input [5]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17, d_date_sk#18] +Arguments: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16], [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] + +(23) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#19] + +(24) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] +Right output [1]: [s_store_sk#19] +Arguments: [sr_store_sk#14], [s_store_sk#19], Inner, BuildRight + +(25) CometProject +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Arguments: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19], [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] + +(26) CometHashAggregate +Input [3]: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Keys [1]: [s_store_sk#19] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#15)), partial_sum(UnscaledValue(sr_net_loss#16))] + +(27) CometExchange +Input [3]: [s_store_sk#19, sum#20, sum#21] +Arguments: hashpartitioning(s_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] + +(29) HashAggregate [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Keys [1]: [s_store_sk#19] +Functions [2]: [sum(UnscaledValue(sr_return_amt#15)), sum(UnscaledValue(sr_net_loss#16))] +Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#15))#22, sum(UnscaledValue(sr_net_loss#16))#23] +Results [3]: [s_store_sk#19, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#22,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(sr_net_loss#16))#23,17,2) AS profit_loss#25] + +(30) BroadcastExchange +Input [3]: [s_store_sk#19, returns#24, profit_loss#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(31) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#19] +Join type: LeftOuter +Join condition: None + +(32) Project [codegen id : 2] +Output [5]: [store channel AS channel#26, s_store_sk#7 AS id#27, sales#12, coalesce(returns#24, 0.00) AS returns#28, (profit#13 - coalesce(profit_loss#25, 0.00)) AS profit#29] +Input [6]: [s_store_sk#7, sales#12, profit#13, s_store_sk#19, returns#24, profit_loss#25] + +(33) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +ReadSchema: struct + +(34) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#34] + +(35) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [1]: [d_date_sk#34] +Arguments: [cs_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight + +(36) CometProject +Input [5]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, d_date_sk#34] +Arguments: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32], [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] + +(37) CometHashAggregate +Input [3]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(UnscaledValue(cs_net_profit#32))] + +(38) CometExchange +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Arguments: hashpartitioning(cs_call_center_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometColumnarToRow [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] + +(40) HashAggregate [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(UnscaledValue(cs_net_profit#32))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#31))#37, sum(UnscaledValue(cs_net_profit#32))#38] +Results [3]: [cs_call_center_sk#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#31))#37,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#32))#38,17,2) AS profit#40] + +(41) BroadcastExchange +Input [3]: [cs_call_center_sk#30, sales#39, profit#40] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(42) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] +ReadSchema: struct + +(43) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#44] + +(44) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Right output [1]: [d_date_sk#44] +Arguments: [cr_returned_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(45) CometProject +Input [4]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43, d_date_sk#44] +Arguments: [cr_return_amount#41, cr_net_loss#42], [cr_return_amount#41, cr_net_loss#42] + +(46) CometHashAggregate +Input [2]: [cr_return_amount#41, cr_net_loss#42] +Keys: [] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#41)), partial_sum(UnscaledValue(cr_net_loss#42))] + +(47) CometExchange +Input [2]: [sum#45, sum#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(48) CometColumnarToRow +Input [2]: [sum#45, sum#46] + +(49) HashAggregate +Input [2]: [sum#45, sum#46] +Keys: [] +Functions [2]: [sum(UnscaledValue(cr_return_amount#41)), sum(UnscaledValue(cr_net_loss#42))] +Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#41))#47, sum(UnscaledValue(cr_net_loss#42))#48] +Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#41))#47,17,2) AS returns#49, MakeDecimal(sum(UnscaledValue(cr_net_loss#42))#48,17,2) AS profit_loss#50] + +(50) BroadcastNestedLoopJoin [codegen id : 4] +Join type: Inner +Join condition: None + +(51) Project [codegen id : 4] +Output [5]: [catalog channel AS channel#51, cs_call_center_sk#30 AS id#52, sales#39, returns#49, (profit#40 - profit_loss#50) AS profit#53] +Input [5]: [cs_call_center_sk#30, sales#39, profit#40, returns#49, profit_loss#50] + +(52) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#57)] +PushedFilters: [IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(53) CometFilter +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Condition : isnotnull(ws_web_page_sk#54) + +(54) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(55) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [ws_sold_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(56) CometProject +Input [5]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57, d_date_sk#58] +Arguments: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56], [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] + +(57) CometNativeScan parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(58) CometFilter +Input [1]: [wp_web_page_sk#59] +Condition : isnotnull(wp_web_page_sk#59) + +(59) CometBroadcastExchange +Input [1]: [wp_web_page_sk#59] +Arguments: [wp_web_page_sk#59] + +(60) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [ws_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(61) CometProject +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Arguments: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59], [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] + +(62) CometHashAggregate +Input [3]: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#55)), partial_sum(UnscaledValue(ws_net_profit#56))] + +(63) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(64) CometColumnarToRow [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] + +(65) HashAggregate [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#55)), sum(UnscaledValue(ws_net_profit#56))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#55))#62, sum(UnscaledValue(ws_net_profit#56))#63] +Results [3]: [wp_web_page_sk#59, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#55))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#56))#63,17,2) AS profit#65] + +(66) CometNativeScan parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#69)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct + +(67) CometFilter +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Condition : isnotnull(wr_web_page_sk#66) + +(68) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#70] + +(69) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Right output [1]: [d_date_sk#70] +Arguments: [wr_returned_date_sk#69], [d_date_sk#70], Inner, BuildRight + +(70) CometProject +Input [5]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69, d_date_sk#70] +Arguments: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68], [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] + +(71) ReusedExchange [Reuses operator id: 59] +Output [1]: [wp_web_page_sk#71] + +(72) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] +Right output [1]: [wp_web_page_sk#71] +Arguments: [wr_web_page_sk#66], [wp_web_page_sk#71], Inner, BuildRight + +(73) CometProject +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Arguments: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71], [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] + +(74) CometHashAggregate +Input [3]: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#67)), partial_sum(UnscaledValue(wr_net_loss#68))] + +(75) CometExchange +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Arguments: hashpartitioning(wp_web_page_sk#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(76) CometColumnarToRow [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] + +(77) HashAggregate [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [sum(UnscaledValue(wr_return_amt#67)), sum(UnscaledValue(wr_net_loss#68))] +Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#67))#74, sum(UnscaledValue(wr_net_loss#68))#75] +Results [3]: [wp_web_page_sk#71, MakeDecimal(sum(UnscaledValue(wr_return_amt#67))#74,17,2) AS returns#76, MakeDecimal(sum(UnscaledValue(wr_net_loss#68))#75,17,2) AS profit_loss#77] + +(78) BroadcastExchange +Input [3]: [wp_web_page_sk#71, returns#76, profit_loss#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(79) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [wp_web_page_sk#59] +Right keys [1]: [wp_web_page_sk#71] +Join type: LeftOuter +Join condition: None + +(80) Project [codegen id : 6] +Output [5]: [web channel AS channel#78, wp_web_page_sk#59 AS id#79, sales#64, coalesce(returns#76, 0.00) AS returns#80, (profit#65 - coalesce(profit_loss#77, 0.00)) AS profit#81] +Input [6]: [wp_web_page_sk#59, sales#64, profit#65, wp_web_page_sk#71, returns#76, profit_loss#77] + +(81) Union + +(82) HashAggregate [codegen id : 7] +Input [5]: [channel#26, id#27, sales#12, returns#28, profit#29] +Keys [2]: [channel#26, id#27] +Functions [3]: [partial_sum(sales#12), partial_sum(returns#28), partial_sum(profit#29)] +Aggregate Attributes [6]: [sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Results [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(83) CometColumnarExchange +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Arguments: hashpartitioning(channel#26, id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(84) CometColumnarToRow [codegen id : 8] +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(85) HashAggregate [codegen id : 8] +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#26, id#27] +Functions [3]: [sum(sales#12), sum(returns#28), sum(profit#29)] +Aggregate Attributes [3]: [sum(sales#12)#94, sum(returns#28)#95, sum(profit#29)#96] +Results [5]: [channel#26, id#27, cast(sum(sales#12)#94 as decimal(37,2)) AS sales#97, cast(sum(returns#28)#95 as decimal(37,2)) AS returns#98, cast(sum(profit#29)#96 as decimal(38,2)) AS profit#99] + +(86) ReusedExchange [Reuses operator id: 83] +Output [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] + +(87) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] + +(88) HashAggregate [codegen id : 16] +Input [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] +Keys [2]: [channel#100, id#101] +Functions [3]: [sum(sales#108), sum(returns#109), sum(profit#110)] +Aggregate Attributes [3]: [sum(sales#108)#94, sum(returns#109)#95, sum(profit#110)#96] +Results [4]: [channel#100, sum(sales#108)#94 AS sales#111, sum(returns#109)#95 AS returns#112, sum(profit#110)#96 AS profit#113] + +(89) HashAggregate [codegen id : 16] +Input [4]: [channel#100, sales#111, returns#112, profit#113] +Keys [1]: [channel#100] +Functions [3]: [partial_sum(sales#111), partial_sum(returns#112), partial_sum(profit#113)] +Aggregate Attributes [6]: [sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] +Results [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] + +(90) CometColumnarExchange +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Arguments: hashpartitioning(channel#100, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] + +(91) CometColumnarToRow [codegen id : 17] +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] + +(92) HashAggregate [codegen id : 17] +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Keys [1]: [channel#100] +Functions [3]: [sum(sales#111), sum(returns#112), sum(profit#113)] +Aggregate Attributes [3]: [sum(sales#111)#126, sum(returns#112)#127, sum(profit#113)#128] +Results [5]: [channel#100, null AS id#129, sum(sales#111)#126 AS sales#130, sum(returns#112)#127 AS returns#131, sum(profit#113)#128 AS profit#132] + +(93) ReusedExchange [Reuses operator id: 83] +Output [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(94) CometColumnarToRow [codegen id : 25] +Input [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(95) HashAggregate [codegen id : 25] +Input [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Keys [2]: [channel#133, id#134] +Functions [3]: [sum(sales#141), sum(returns#142), sum(profit#143)] +Aggregate Attributes [3]: [sum(sales#141)#94, sum(returns#142)#95, sum(profit#143)#96] +Results [3]: [sum(sales#141)#94 AS sales#144, sum(returns#142)#95 AS returns#145, sum(profit#143)#96 AS profit#146] + +(96) HashAggregate [codegen id : 25] +Input [3]: [sales#144, returns#145, profit#146] +Keys: [] +Functions [3]: [partial_sum(sales#144), partial_sum(returns#145), partial_sum(profit#146)] +Aggregate Attributes [6]: [sum#147, isEmpty#148, sum#149, isEmpty#150, sum#151, isEmpty#152] +Results [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] + +(97) CometColumnarExchange +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] + +(98) CometColumnarToRow [codegen id : 26] +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] + +(99) HashAggregate [codegen id : 26] +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] +Keys: [] +Functions [3]: [sum(sales#144), sum(returns#145), sum(profit#146)] +Aggregate Attributes [3]: [sum(sales#144)#159, sum(returns#145)#160, sum(profit#146)#161] +Results [5]: [null AS channel#162, null AS id#163, sum(sales#144)#159 AS sales#164, sum(returns#145)#160 AS returns#165, sum(profit#146)#161 AS profit#166] + +(100) Union + +(101) HashAggregate [codegen id : 27] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Keys [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#26, id#27, sales#97, returns#98, profit#99] + +(102) CometColumnarExchange +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Arguments: hashpartitioning(channel#26, id#27, sales#97, returns#98, profit#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] + +(103) CometHashAggregate +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Keys [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Functions: [] + +(104) CometTakeOrderedAndProject +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,id#27 ASC NULLS FIRST], output=[channel#26,id#27,sales#97,returns#98,profit#99]), [channel#26, id#27, sales#97, returns#98, profit#99], 100, 0, [channel#26 ASC NULLS FIRST, id#27 ASC NULLS FIRST], [channel#26, id#27, sales#97, returns#98, profit#99] + +(105) CometColumnarToRow [codegen id : 28] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/extended.txt new file mode 100644 index 0000000000..7173668874 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/extended.txt @@ -0,0 +1,328 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- Project + : : +- BroadcastHashJoin + : : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#1, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#3,17,2) AS sales#4, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#6,17,2) AS profit#7)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#8, MakeDecimal(sum(UnscaledValue(sr_return_amt#9))#10,17,2) AS returns#11, MakeDecimal(sum(UnscaledValue(sr_net_loss#12))#13,17,2) AS profit_loss#14)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_call_center_sk#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#16))#17,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(cs_net_profit#19))#20,17,2) AS profit#21)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(cr_return_amount#22))#23,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(cr_net_loss#25))#26,17,2) AS profit_loss#27)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#28, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#29))#30,17,2) AS sales#31, MakeDecimal(sum(UnscaledValue(ws_net_profit#32))#33,17,2) AS profit#34)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- BroadcastExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#35, MakeDecimal(sum(UnscaledValue(wr_return_amt#36))#37,17,2) AS returns#38, MakeDecimal(sum(UnscaledValue(wr_net_loss#39))#40,17,2) AS profit_loss#41)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- Project + : : +- BroadcastHashJoin + : : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#1, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#3,17,2) AS sales#4, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#6,17,2) AS profit#7)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#8, MakeDecimal(sum(UnscaledValue(sr_return_amt#9))#10,17,2) AS returns#11, MakeDecimal(sum(UnscaledValue(sr_net_loss#12))#13,17,2) AS profit_loss#14)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_call_center_sk#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#16))#17,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(cs_net_profit#19))#20,17,2) AS profit#21)] + : : : +- CometColumnarToRow + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(cr_return_amount#22))#23,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(cr_net_loss#25))#26,17,2) AS profit_loss#27)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#28, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#29))#30,17,2) AS sales#31, MakeDecimal(sum(UnscaledValue(ws_net_profit#32))#33,17,2) AS profit#34)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- BroadcastExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#35, MakeDecimal(sum(UnscaledValue(wr_return_amt#36))#37,17,2) AS returns#38, MakeDecimal(sum(UnscaledValue(wr_net_loss#39))#40,17,2) AS profit_loss#41)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- Project + : +- BroadcastHashJoin + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#1, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#3,17,2) AS sales#4, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#6,17,2) AS profit#7)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(s_store_sk#8, MakeDecimal(sum(UnscaledValue(sr_return_amt#9))#10,17,2) AS returns#11, MakeDecimal(sum(UnscaledValue(sr_net_loss#12))#13,17,2) AS profit_loss#14)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- Project + : +- BroadcastNestedLoopJoin + : :- BroadcastExchange + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cs_call_center_sk#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#16))#17,17,2) AS sales#18, MakeDecimal(sum(UnscaledValue(cs_net_profit#19))#20,17,2) AS profit#21)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(cr_return_amount#22))#23,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(cr_net_loss#25))#26,17,2) AS profit_loss#27)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- BroadcastHashJoin + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#28, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#29))#30,17,2) AS sales#31, MakeDecimal(sum(UnscaledValue(ws_net_profit#32))#33,17,2) AS profit#34)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- BroadcastExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(wp_web_page_sk#35, MakeDecimal(sum(UnscaledValue(wr_return_amt#36))#37,17,2) AS returns#38, MakeDecimal(sum(UnscaledValue(wr_net_loss#39))#40,17,2) AS profit_loss#41)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 242 out of 302 eligible operators (80%). Final plan contains 24 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..bc67668c0d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_datafusion/simplified.txt @@ -0,0 +1,136 @@ +WholeStageCodegen (28) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (27) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (8) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (7) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + Project [s_store_sk,sales,returns,profit,profit_loss] + BroadcastHashJoin [s_store_sk,s_store_sk] + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_sk] #3 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_sk] #7 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (4) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (3) + HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cs_call_center_sk] #9 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange #10 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (6) + Project [wp_web_page_sk,sales,returns,profit,profit_loss] + BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #11 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (5) + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #14 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 + WholeStageCodegen (17) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #15 + WholeStageCodegen (16) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (26) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #16 + WholeStageCodegen (25) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/explain.txt new file mode 100644 index 0000000000..5ac0172cde --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/explain.txt @@ -0,0 +1,389 @@ +== Physical Plan == +TakeOrderedAndProject (69) ++- * Project (68) + +- * SortMergeJoin Inner (67) + :- * Project (45) + : +- * SortMergeJoin Inner (44) + : :- * Sort (22) + : : +- * HashAggregate (21) + : : +- * CometColumnarToRow (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : +- * Sort (43) + : +- * Filter (42) + : +- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (34) + : : +- CometFilter (33) + : : +- CometSortMergeJoin (32) + : : :- CometSort (26) + : : : +- CometExchange (25) + : : : +- CometFilter (24) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (23) + : : +- CometSort (31) + : : +- CometExchange (30) + : : +- CometProject (29) + : : +- CometFilter (28) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (27) + : +- ReusedExchange (35) + +- * Sort (66) + +- * Filter (65) + +- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometFilter (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometFilter (47) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (46) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.catalog_returns (50) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) + +(3) CometExchange +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] + +(5) CometNativeScan parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) + +(7) CometProject +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] + +(8) CometExchange +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter + +(11) CometFilter +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) + +(12) CometProject +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] + +(13) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) + +(15) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(16) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight + +(17) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] + +(18) CometHashAggregate +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] +Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] + +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(20) CometColumnarToRow [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] + +(21) HashAggregate [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] +Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] +Aggregate Attributes [3]: [sum(ss_quantity#4)#16, sum(UnscaledValue(ss_wholesale_cost#5))#17, sum(UnscaledValue(ss_sales_price#6))#18] +Results [6]: [d_year#12 AS ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#16 AS ss_qty#20, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#17,17,2) AS ss_wc#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#18,17,2) AS ss_sp#22] + +(22) Sort [codegen id : 1] +Input [6]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 + +(23) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(24) CometFilter +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_item_sk#23) AND isnotnull(ws_bill_customer_sk#24)) + +(25) CometExchange +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: hashpartitioning(ws_order_number#25, ws_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometSort +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_order_number#25 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST] + +(27) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Condition : (isnotnull(wr_order_number#31) AND isnotnull(wr_item_sk#30)) + +(29) CometProject +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_item_sk#30, wr_order_number#31] + +(30) CometExchange +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: hashpartitioning(wr_order_number#31, wr_item_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(31) CometSort +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_order_number#31 ASC NULLS FIRST, wr_item_sk#30 ASC NULLS FIRST] + +(32) CometSortMergeJoin +Left output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [ws_order_number#25, ws_item_sk#23], [wr_order_number#31, wr_item_sk#30], LeftOuter + +(33) CometFilter +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Condition : isnull(wr_order_number#31) + +(34) CometProject +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] + +(35) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#33, d_year#34] + +(36) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [d_date_sk#33, d_year#34] +Arguments: [ws_sold_date_sk#29], [d_date_sk#33], Inner, BuildRight + +(37) CometProject +Input [8]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, d_date_sk#33, d_year#34] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] + +(38) CometHashAggregate +Input [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [partial_sum(ws_quantity#26), partial_sum(UnscaledValue(ws_wholesale_cost#27)), partial_sum(UnscaledValue(ws_sales_price#28))] + +(39) CometExchange +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Arguments: hashpartitioning(d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(40) CometColumnarToRow [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] + +(41) HashAggregate [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +Aggregate Attributes [3]: [sum(ws_quantity#26)#38, sum(UnscaledValue(ws_wholesale_cost#27))#39, sum(UnscaledValue(ws_sales_price#28))#40] +Results [6]: [d_year#34 AS ws_sold_year#41, ws_item_sk#23, ws_bill_customer_sk#24 AS ws_customer_sk#42, sum(ws_quantity#26)#38 AS ws_qty#43, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#27))#39,17,2) AS ws_wc#44, MakeDecimal(sum(UnscaledValue(ws_sales_price#28))#40,17,2) AS ws_sp#45] + +(42) Filter [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Condition : (coalesce(ws_qty#43, 0) > 0) + +(43) Sort [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Arguments: [ws_sold_year#41 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST, ws_customer_sk#42 ASC NULLS FIRST], false, 0 + +(44) SortMergeJoin [codegen id : 3] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42] +Join type: Inner +Join condition: None + +(45) Project [codegen id : 3] +Output [9]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45] +Input [12]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] + +(46) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#52)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(47) CometFilter +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Condition : (isnotnull(cs_item_sk#47) AND isnotnull(cs_bill_customer_sk#46)) + +(48) CometExchange +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: hashpartitioning(cs_order_number#48, cs_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(49) CometSort +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_order_number#48 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST] + +(50) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Condition : (isnotnull(cr_order_number#54) AND isnotnull(cr_item_sk#53)) + +(52) CometProject +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_item_sk#53, cr_order_number#54] + +(53) CometExchange +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: hashpartitioning(cr_order_number#54, cr_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(54) CometSort +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_order_number#54 ASC NULLS FIRST, cr_item_sk#53 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cs_order_number#48, cs_item_sk#47], [cr_order_number#54, cr_item_sk#53], LeftOuter + +(56) CometFilter +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Condition : isnull(cr_order_number#54) + +(57) CometProject +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] + +(58) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#56, d_year#57] + +(59) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [d_date_sk#56, d_year#57] +Arguments: [cs_sold_date_sk#52], [d_date_sk#56], Inner, BuildRight + +(60) CometProject +Input [8]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, d_date_sk#56, d_year#57] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] + +(61) CometHashAggregate +Input [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [partial_sum(cs_quantity#49), partial_sum(UnscaledValue(cs_wholesale_cost#50)), partial_sum(UnscaledValue(cs_sales_price#51))] + +(62) CometExchange +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Arguments: hashpartitioning(d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(63) CometColumnarToRow [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] + +(64) HashAggregate [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [sum(cs_quantity#49), sum(UnscaledValue(cs_wholesale_cost#50)), sum(UnscaledValue(cs_sales_price#51))] +Aggregate Attributes [3]: [sum(cs_quantity#49)#61, sum(UnscaledValue(cs_wholesale_cost#50))#62, sum(UnscaledValue(cs_sales_price#51))#63] +Results [6]: [d_year#57 AS cs_sold_year#64, cs_item_sk#47, cs_bill_customer_sk#46 AS cs_customer_sk#65, sum(cs_quantity#49)#61 AS cs_qty#66, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#50))#62,17,2) AS cs_wc#67, MakeDecimal(sum(UnscaledValue(cs_sales_price#51))#63,17,2) AS cs_sp#68] + +(65) Filter [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Condition : (coalesce(cs_qty#66, 0) > 0) + +(66) Sort [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Arguments: [cs_sold_year#64 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST, cs_customer_sk#65 ASC NULLS FIRST], false, 0 + +(67) SortMergeJoin [codegen id : 5] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65] +Join type: Inner +Join condition: None + +(68) Project [codegen id : 5] +Output [13]: [round((cast(ss_qty#20 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)))), 2) AS ratio#69, ss_qty#20 AS store_qty#70, ss_wc#21 AS store_wholesale_cost#71, ss_sp#22 AS store_sales_price#72, (coalesce(ws_qty#43, 0) + coalesce(cs_qty#66, 0)) AS other_chan_qty#73, (coalesce(ws_wc#44, 0.00) + coalesce(cs_wc#67, 0.00)) AS other_chan_wholesale_cost#74, (coalesce(ws_sp#45, 0.00) + coalesce(cs_sp#68, 0.00)) AS other_chan_sales_price#75, ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Input [15]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45, cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] + +(69) TakeOrderedAndProject +Input [13]: [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75, ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: 100, [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#20 DESC NULLS LAST, ss_wc#21 DESC NULLS LAST, ss_sp#22 DESC NULLS LAST, other_chan_qty#73 ASC NULLS FIRST, other_chan_wholesale_cost#74 ASC NULLS FIRST, other_chan_sales_price#75 ASC NULLS FIRST, ratio#69 ASC NULLS FIRST], [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/extended.txt new file mode 100644 index 0000000000..52deb08088 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/extended.txt @@ -0,0 +1,75 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- SortMergeJoin + :- Project + : +- SortMergeJoin + : :- Sort + : : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#1 AS ss_sold_year#2, ss_item_sk#3, ss_customer_sk#4, sum(ss_quantity#5)#6 AS ss_qty#7, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#8))#9,17,2) AS ss_wc#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#11))#12,17,2) AS ss_sp#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- Sort + : +- Filter + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#14 AS ws_sold_year#15, ws_item_sk#16, ws_bill_customer_sk#17 AS ws_customer_sk#18, sum(ws_quantity#19)#20 AS ws_qty#21, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#22))#23,17,2) AS ws_wc#24, MakeDecimal(sum(UnscaledValue(ws_sales_price#25))#26,17,2) AS ws_sp#27)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Sort + +- Filter + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(d_year#28 AS cs_sold_year#29, cs_item_sk#30, cs_bill_customer_sk#31 AS cs_customer_sk#32, sum(cs_quantity#33)#34 AS cs_qty#35, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#36))#37,17,2) AS cs_wc#38, MakeDecimal(sum(UnscaledValue(cs_sales_price#39))#40,17,2) AS cs_sp#41)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 57 out of 70 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/simplified.txt new file mode 100644 index 0000000000..42d266efd6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_datafusion/simplified.txt @@ -0,0 +1,81 @@ +TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] + WholeStageCodegen (5) + Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] + SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] + InputAdapter + WholeStageCodegen (3) + Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] + SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] + InputAdapter + WholeStageCodegen (1) + Sort [ss_sold_year,ss_item_sk,ss_customer_sk] + HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + WholeStageCodegen (2) + Sort [ws_sold_year,ws_item_sk,ws_customer_sk] + Filter [ws_qty] + HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + WholeStageCodegen (4) + Sort [cs_sold_year,cs_item_sk,cs_customer_sk] + Filter [cs_qty] + HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/explain.txt new file mode 100644 index 0000000000..6e6cb674d3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/explain.txt @@ -0,0 +1,696 @@ +== Physical Plan == +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometHashAggregate (122) + +- CometColumnarExchange (121) + +- * HashAggregate (120) + +- Union (119) + :- * HashAggregate (104) + : +- * CometColumnarToRow (103) + : +- CometColumnarExchange (102) + : +- * HashAggregate (101) + : +- Union (100) + : :- * HashAggregate (39) + : : +- * CometColumnarToRow (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (29) + : : : +- CometBroadcastHashJoin (28) + : : : :- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometProject (17) + : : : : : +- CometBroadcastHashJoin (16) + : : : : : :- CometProject (11) + : : : : : : +- CometSortMergeJoin (10) + : : : : : : :- CometSort (4) + : : : : : : : +- CometExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometSort (9) + : : : : : : +- CometExchange (8) + : : : : : : +- CometProject (7) + : : : : : : +- CometFilter (6) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : : : : +- CometBroadcastExchange (15) + : : : : : +- CometProject (14) + : : : : : +- CometFilter (13) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.store (18) + : : : +- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.item (24) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometNativeScan parquet spark_catalog.default.promotion (30) + : :- * HashAggregate (69) + : : +- * CometColumnarToRow (68) + : : +- CometExchange (67) + : : +- CometHashAggregate (66) + : : +- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (62) + : : : +- CometBroadcastHashJoin (61) + : : : :- CometProject (59) + : : : : +- CometBroadcastHashJoin (58) + : : : : :- CometProject (53) + : : : : : +- CometBroadcastHashJoin (52) + : : : : : :- CometProject (50) + : : : : : : +- CometSortMergeJoin (49) + : : : : : : :- CometSort (43) + : : : : : : : +- CometExchange (42) + : : : : : : : +- CometFilter (41) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (40) + : : : : : : +- CometSort (48) + : : : : : : +- CometExchange (47) + : : : : : : +- CometProject (46) + : : : : : : +- CometFilter (45) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (44) + : : : : : +- ReusedExchange (51) + : : : : +- CometBroadcastExchange (57) + : : : : +- CometProject (56) + : : : : +- CometFilter (55) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (54) + : : : +- ReusedExchange (60) + : : +- ReusedExchange (63) + : +- * HashAggregate (99) + : +- * CometColumnarToRow (98) + : +- CometExchange (97) + : +- CometHashAggregate (96) + : +- CometProject (95) + : +- CometBroadcastHashJoin (94) + : :- CometProject (92) + : : +- CometBroadcastHashJoin (91) + : : :- CometProject (89) + : : : +- CometBroadcastHashJoin (88) + : : : :- CometProject (83) + : : : : +- CometBroadcastHashJoin (82) + : : : : :- CometProject (80) + : : : : : +- CometSortMergeJoin (79) + : : : : : :- CometSort (73) + : : : : : : +- CometExchange (72) + : : : : : : +- CometFilter (71) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (70) + : : : : : +- CometSort (78) + : : : : : +- CometExchange (77) + : : : : : +- CometProject (76) + : : : : : +- CometFilter (75) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (74) + : : : : +- ReusedExchange (81) + : : : +- CometBroadcastExchange (87) + : : : +- CometProject (86) + : : : +- CometFilter (85) + : : : +- CometNativeScan parquet spark_catalog.default.web_site (84) + : : +- ReusedExchange (90) + : +- ReusedExchange (93) + :- * HashAggregate (111) + : +- * CometColumnarToRow (110) + : +- CometColumnarExchange (109) + : +- * HashAggregate (108) + : +- * HashAggregate (107) + : +- * CometColumnarToRow (106) + : +- ReusedExchange (105) + +- * HashAggregate (118) + +- * CometColumnarToRow (117) + +- CometColumnarExchange (116) + +- * HashAggregate (115) + +- * HashAggregate (114) + +- * CometColumnarToRow (113) + +- ReusedExchange (112) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) + +(3) CometExchange +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] + +(5) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(6) CometFilter +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) + +(7) CometProject +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] + +(8) CometExchange +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter + +(11) CometProject +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] + +(12) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) + +(14) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(15) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(16) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight + +(17) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] + +(18) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#15, s_store_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) + +(20) CometProject +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#16, 16)) AS s_store_id#17] + +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight + +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) + +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] + +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] + +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(30) CometNativeScan parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((isnotnull(p_channel_tv#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#21, 1)) = N)) AND isnotnull(p_promo_sk#20)) + +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] + +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] + +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight + +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] + +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] + +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [store channel AS channel#30, concat(store, s_store_id#17) AS id#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#32, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#33, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#34] + +(40) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct + +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) + +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] + +(44) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) + +(46) CometProject +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] + +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] + +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter + +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] + +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] + +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight + +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] + +(54) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#49, 16)) AS cp_catalog_page_id#50] + +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] + +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight + +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] + +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight + +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] + +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight + +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] + +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] + +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#50) AS id#62, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#64, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#65] + +(70) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) + +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] + +(74) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] +ReadSchema: struct + +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) + +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] + +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] + +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter + +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] + +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] + +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight + +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] + +(84) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) + +(86) CometProject +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#80, 16)) AS web_site_id#81] + +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] + +(88) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight + +(89) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] + +(91) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight + +(92) CometProject +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#81) AS id#93, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#94, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#95, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#96] + +(100) Union + +(101) HashAggregate [codegen id : 4] +Input [5]: [channel#30, id#31, sales#32, returns#33, profit#34] +Keys [2]: [channel#30, id#31] +Functions [3]: [partial_sum(sales#32), partial_sum(returns#33), partial_sum(profit#34)] +Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Results [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(102) CometColumnarExchange +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(103) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(104) HashAggregate [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [5]: [channel#30, id#31, cast(sum(sales#32)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#33)#110 as decimal(38,2)) AS returns#113, cast(sum(profit#34)#111 as decimal(38,2)) AS profit#114] + +(105) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] + +(106) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] + +(107) HashAggregate [codegen id : 10] +Input [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] +Keys [2]: [channel#115, id#116] +Functions [3]: [sum(sales#123), sum(returns#124), sum(profit#125)] +Aggregate Attributes [3]: [sum(sales#123)#109, sum(returns#124)#110, sum(profit#125)#111] +Results [4]: [channel#115, sum(sales#123)#109 AS sales#126, sum(returns#124)#110 AS returns#127, sum(profit#125)#111 AS profit#128] + +(108) HashAggregate [codegen id : 10] +Input [4]: [channel#115, sales#126, returns#127, profit#128] +Keys [1]: [channel#115] +Functions [3]: [partial_sum(sales#126), partial_sum(returns#127), partial_sum(profit#128)] +Aggregate Attributes [6]: [sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134] +Results [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(109) CometColumnarExchange +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Arguments: hashpartitioning(channel#115, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] + +(110) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(111) HashAggregate [codegen id : 11] +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Keys [1]: [channel#115] +Functions [3]: [sum(sales#126), sum(returns#127), sum(profit#128)] +Aggregate Attributes [3]: [sum(sales#126)#141, sum(returns#127)#142, sum(profit#128)#143] +Results [5]: [channel#115, null AS id#144, sum(sales#126)#141 AS sales#145, sum(returns#127)#142 AS returns#146, sum(profit#128)#143 AS profit#147] + +(112) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] + +(113) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] + +(114) HashAggregate [codegen id : 16] +Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] +Keys [2]: [channel#148, id#149] +Functions [3]: [sum(sales#156), sum(returns#157), sum(profit#158)] +Aggregate Attributes [3]: [sum(sales#156)#109, sum(returns#157)#110, sum(profit#158)#111] +Results [3]: [sum(sales#156)#109 AS sales#159, sum(returns#157)#110 AS returns#160, sum(profit#158)#111 AS profit#161] + +(115) HashAggregate [codegen id : 16] +Input [3]: [sales#159, returns#160, profit#161] +Keys: [] +Functions [3]: [partial_sum(sales#159), partial_sum(returns#160), partial_sum(profit#161)] +Aggregate Attributes [6]: [sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167] +Results [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] + +(116) CometColumnarExchange +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] + +(117) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] + +(118) HashAggregate [codegen id : 17] +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] +Keys: [] +Functions [3]: [sum(sales#159), sum(returns#160), sum(profit#161)] +Aggregate Attributes [3]: [sum(sales#159)#174, sum(returns#160)#175, sum(profit#161)#176] +Results [5]: [null AS channel#177, null AS id#178, sum(sales#159)#174 AS sales#179, sum(returns#160)#175 AS returns#180, sum(profit#161)#176 AS profit#181] + +(119) Union + +(120) HashAggregate [codegen id : 18] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#30, id#31, sales#112, returns#113, profit#114] + +(121) CometColumnarExchange +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#30, id#31, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] + +(122) CometHashAggregate +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Functions: [] + +(123) CometTakeOrderedAndProject +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#30 ASC NULLS FIRST,id#31 ASC NULLS FIRST], output=[channel#30,id#31,sales#112,returns#113,profit#114]), [channel#30, id#31, sales#112, returns#113, profit#114], 100, 0, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#112, returns#113, profit#114] + +(124) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/extended.txt new file mode 100644 index 0000000000..8490c83af9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/extended.txt @@ -0,0 +1,382 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(store channel AS channel#1, concat(store, s_store_id#2) AS id#3, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#4))#5,17,2) AS sales#6, sum(coalesce(cast(sr_return_amt#7 as decimal(12,2)), 0.00))#8 AS returns#9, sum((ss_net_profit#10 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#12 AS profit#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(catalog channel AS channel#14, concat(catalog_page, cp_catalog_page_id#15) AS id#16, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#17))#18,17,2) AS sales#19, sum(coalesce(cast(cr_return_amount#20 as decimal(12,2)), 0.00))#21 AS returns#22, sum((cs_net_profit#23 - coalesce(cast(cr_net_loss#24 as decimal(12,2)), 0.00)))#25 AS profit#26)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(web channel AS channel#27, concat(web_site, web_site_id#28) AS id#29, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#30))#31,17,2) AS sales#32, sum(coalesce(cast(wr_return_amt#33 as decimal(12,2)), 0.00))#34 AS returns#35, sum((ws_net_profit#36 - coalesce(cast(wr_net_loss#37 as decimal(12,2)), 0.00)))#38 AS profit#39)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_site + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(store channel AS channel#1, concat(store, s_store_id#2) AS id#3, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#4))#5,17,2) AS sales#6, sum(coalesce(cast(sr_return_amt#7 as decimal(12,2)), 0.00))#8 AS returns#9, sum((ss_net_profit#10 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#12 AS profit#13)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(catalog channel AS channel#14, concat(catalog_page, cp_catalog_page_id#15) AS id#16, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#17))#18,17,2) AS sales#19, sum(coalesce(cast(cr_return_amount#20 as decimal(12,2)), 0.00))#21 AS returns#22, sum((cs_net_profit#23 - coalesce(cast(cr_net_loss#24 as decimal(12,2)), 0.00)))#25 AS profit#26)] + : : +- CometColumnarToRow + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(web channel AS channel#27, concat(web_site, web_site_id#28) AS id#29, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#30))#31,17,2) AS sales#32, sum(coalesce(cast(wr_return_amt#33 as decimal(12,2)), 0.00))#34 AS returns#35, sum((ws_net_profit#36 - coalesce(cast(wr_net_loss#37 as decimal(12,2)), 0.00)))#38 AS profit#39)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_site + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(store channel AS channel#1, concat(store, s_store_id#2) AS id#3, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#4))#5,17,2) AS sales#6, sum(coalesce(cast(sr_return_amt#7 as decimal(12,2)), 0.00))#8 AS returns#9, sum((ss_net_profit#10 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#12 AS profit#13)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(catalog channel AS channel#14, concat(catalog_page, cp_catalog_page_id#15) AS id#16, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#17))#18,17,2) AS sales#19, sum(coalesce(cast(cr_return_amount#20 as decimal(12,2)), 0.00))#21 AS returns#22, sum((cs_net_profit#23 - coalesce(cast(cr_net_loss#24 as decimal(12,2)), 0.00)))#25 AS profit#26)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(web channel AS channel#27, concat(web_site, web_site_id#28) AS id#29, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#30))#31,17,2) AS sales#32, sum(coalesce(cast(wr_return_amt#33 as decimal(12,2)), 0.00))#34 AS returns#35, sum((ws_net_profit#36 - coalesce(cast(wr_net_loss#37 as decimal(12,2)), 0.00)))#38 AS profit#39)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_site + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 341 out of 365 eligible operators (93%). Final plan contains 15 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..2d0d1355e3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_datafusion/simplified.txt @@ -0,0 +1,146 @@ +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #14 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #18 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #19 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/explain.txt new file mode 100644 index 0000000000..e6ecb075c9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/explain.txt @@ -0,0 +1,241 @@ +== Physical Plan == +TakeOrderedAndProject (42) ++- * Project (41) + +- Window (40) + +- * CometColumnarToRow (39) + +- CometSort (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometColumnarExchange (35) + +- * HashAggregate (34) + +- Union (33) + :- * HashAggregate (18) + : +- * CometColumnarToRow (17) + : +- CometExchange (16) + : +- CometHashAggregate (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + :- * HashAggregate (25) + : +- * CometColumnarToRow (24) + : +- CometColumnarExchange (23) + : +- * HashAggregate (22) + : +- * HashAggregate (21) + : +- * CometColumnarToRow (20) + : +- ReusedExchange (19) + +- * HashAggregate (32) + +- * CometColumnarToRow (31) + +- CometColumnarExchange (30) + +- * HashAggregate (29) + +- * HashAggregate (28) + +- * CometColumnarToRow (27) + +- ReusedExchange (26) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] +Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] + +(9) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#6, i_class#7, i_category#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) + +(11) CometProject +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#7, 50)) AS i_class#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#8, 50)) AS i_category#10] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#1, ws_net_paid#2] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] + +(15) CometHashAggregate +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] + +(16) CometExchange +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [3]: [i_category#10, i_class#9, sum#11] + +(18) HashAggregate [codegen id : 1] +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#12] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#12,17,2) as decimal(27,2)) AS total_sum#13, i_category#10 AS i_category#14, i_class#9 AS i_class#15, 0 AS g_category#16, 0 AS g_class#17, 0 AS lochierarchy#18] + +(19) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#19, i_class#20, sum#21] + +(20) CometColumnarToRow [codegen id : 2] +Input [3]: [i_category#19, i_class#20, sum#21] + +(21) HashAggregate [codegen id : 2] +Input [3]: [i_category#19, i_class#20, sum#21] +Keys [2]: [i_category#19, i_class#20] +Functions [1]: [sum(UnscaledValue(ws_net_paid#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#22))#12] +Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#22))#12,17,2) AS total_sum#23, i_category#19] + +(22) HashAggregate [codegen id : 2] +Input [2]: [total_sum#23, i_category#19] +Keys [1]: [i_category#19] +Functions [1]: [partial_sum(total_sum#23)] +Aggregate Attributes [2]: [sum#24, isEmpty#25] +Results [3]: [i_category#19, sum#26, isEmpty#27] + +(23) CometColumnarExchange +Input [3]: [i_category#19, sum#26, isEmpty#27] +Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometColumnarToRow [codegen id : 3] +Input [3]: [i_category#19, sum#26, isEmpty#27] + +(25) HashAggregate [codegen id : 3] +Input [3]: [i_category#19, sum#26, isEmpty#27] +Keys [1]: [i_category#19] +Functions [1]: [sum(total_sum#23)] +Aggregate Attributes [1]: [sum(total_sum#23)#28] +Results [6]: [sum(total_sum#23)#28 AS total_sum#29, i_category#19, null AS i_class#30, 0 AS g_category#31, 1 AS g_class#32, 1 AS lochierarchy#33] + +(26) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#34, i_class#35, sum#36] + +(27) CometColumnarToRow [codegen id : 4] +Input [3]: [i_category#34, i_class#35, sum#36] + +(28) HashAggregate [codegen id : 4] +Input [3]: [i_category#34, i_class#35, sum#36] +Keys [2]: [i_category#34, i_class#35] +Functions [1]: [sum(UnscaledValue(ws_net_paid#37))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#37))#12] +Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#37))#12,17,2) AS total_sum#38] + +(29) HashAggregate [codegen id : 4] +Input [1]: [total_sum#38] +Keys: [] +Functions [1]: [partial_sum(total_sum#38)] +Aggregate Attributes [2]: [sum#39, isEmpty#40] +Results [2]: [sum#41, isEmpty#42] + +(30) CometColumnarExchange +Input [2]: [sum#41, isEmpty#42] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(31) CometColumnarToRow [codegen id : 5] +Input [2]: [sum#41, isEmpty#42] + +(32) HashAggregate [codegen id : 5] +Input [2]: [sum#41, isEmpty#42] +Keys: [] +Functions [1]: [sum(total_sum#38)] +Aggregate Attributes [1]: [sum(total_sum#38)#43] +Results [6]: [sum(total_sum#38)#43 AS total_sum#44, null AS i_category#45, null AS i_class#46, 1 AS g_category#47, 1 AS g_class#48, 2 AS lochierarchy#49] + +(33) Union + +(34) HashAggregate [codegen id : 6] +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Keys [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Functions: [] +Aggregate Attributes: [] +Results [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] + +(35) CometColumnarExchange +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Arguments: hashpartitioning(total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(36) CometHashAggregate +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Keys [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Functions: [] + +(37) CometExchange +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: hashpartitioning(lochierarchy#18, _w0#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(38) CometSort +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50], [lochierarchy#18 ASC NULLS FIRST, _w0#50 ASC NULLS FIRST, total_sum#13 DESC NULLS LAST] + +(39) CometColumnarToRow [codegen id : 7] +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] + +(40) Window +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: [rank(total_sum#13) windowspecdefinition(lochierarchy#18, _w0#50, total_sum#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#51], [lochierarchy#18, _w0#50], [total_sum#13 DESC NULLS LAST] + +(41) Project [codegen id : 8] +Output [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] +Input [6]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50, rank_within_parent#51] + +(42) TakeOrderedAndProject +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] +Arguments: 100, [lochierarchy#18 DESC NULLS LAST, CASE WHEN (lochierarchy#18 = 0) THEN i_category#14 END ASC NULLS FIRST, rank_within_parent#51 ASC NULLS FIRST], [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/extended.txt new file mode 100644 index 0000000000..d4d75da8cd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/extended.txt @@ -0,0 +1,74 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#1))#2,17,2) as decimal(27,2)) AS total_sum#3, i_category#4 AS i_category#5, i_class#6 AS i_class#7, 0 AS g_category#8, 0 AS g_class#9, 0 AS lochierarchy#10)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ws_net_paid#11))#2,17,2) AS total_sum#12, i_category#13)] + : +- CometColumnarToRow + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(MakeDecimal(sum(UnscaledValue(ws_net_paid#14))#2,17,2) AS total_sum#15)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 54 out of 66 eligible operators (81%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/simplified.txt new file mode 100644 index 0000000000..1c49c4f068 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_datafusion/simplified.txt @@ -0,0 +1,58 @@ +TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] + WholeStageCodegen (8) + Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] + InputAdapter + Window [total_sum,lochierarchy,_w0] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [total_sum,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_category] [total_sum,i_category,i_class,lochierarchy,_w0,g_class] + CometColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 + WholeStageCodegen (6) + HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class] #3 + CometHashAggregate [ws_net_paid] [i_category,i_class,sum] + CometProject [ws_net_paid,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + WholeStageCodegen (3) + HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_category] #6 + WholeStageCodegen (2) + HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum] #3 + WholeStageCodegen (5) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #7 + WholeStageCodegen (4) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/explain.txt new file mode 100644 index 0000000000..670f2ff367 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/explain.txt @@ -0,0 +1,146 @@ +== Physical Plan == +* CometColumnarToRow (26) ++- CometSort (25) + +- CometColumnarExchange (24) + +- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_item_sk#1) + +(3) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS _w0#18] + +(19) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] + +(22) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] + +(24) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(25) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] + +(26) CometColumnarToRow [codegen id : 4] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/extended.txt new file mode 100644 index 0000000000..2defeb1407 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/extended.txt @@ -0,0 +1,28 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometColumnarExchange + +- HashAggregate [COMET: makedecimal is not supported, Unsupported result expressions found in: List(i_item_id#1, i_item_desc#2, i_category#3, i_class#4, i_current_price#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#7,17,2) AS itemrevenue#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#7,17,2) AS _w0#9)] + +- CometColumnarToRow + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 20 out of 23 eligible operators (86%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/simplified.txt new file mode 100644 index 0000000000..e5c1501235 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_datafusion/simplified.txt @@ -0,0 +1,34 @@ +WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] + CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 + WholeStageCodegen (3) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometColumnarExchange [i_class] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/explain.txt index 948656e6bb..64025b3367 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/explain.txt @@ -1,258 +1,230 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (28) - : : +- * BroadcastHashJoin LeftSemi BuildRight (27) - : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : +- BroadcastExchange (13) - : : : +- * Project (12) - : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : :- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : +- BroadcastExchange (26) - : : +- Union (25) - : : :- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (22) - : +- BroadcastExchange (33) - : +- * Project (32) - : +- * Filter (31) - : +- * ColumnarToRow (30) - : +- Scan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (39) - +- * Filter (38) - +- * ColumnarToRow (37) - +- Scan parquet spark_catalog.default.customer_demographics (36) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(6) CometProject Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] +(9) CometProject Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#4] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9 AS customer_sk#12] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [customer_sk#11], [ws_bill_customer_sk#9 AS customer_sk#11] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +(16) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(22) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#15] +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#14] -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#13 AS customer_sk#16] -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +(19) CometProject +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [customer_sk#15], [cs_ship_customer_sk#12 AS customer_sk#15] -(25) Union +(20) CometUnion +Child 0 Input [1]: [customer_sk#11] +Child 1 Input [1]: [customer_sk#15] -(26) BroadcastExchange -Input [1]: [customer_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(21) CometBroadcastExchange +Input [1]: [customer_sk#11] +Arguments: [customer_sk#11] -(27) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [customer_sk#12] -Join type: LeftSemi -Join condition: None +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customer_sk#11] +Arguments: [c_customer_sk#1], [customer_sk#11], LeftSemi, BuildRight -(28) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +(23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(29) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_county#18] +(24) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#16, ca_county#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] +(25) CometFilter +Input [2]: [ca_address_sk#16, ca_county#17] +Condition : (ca_county#17 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#16)) -(31) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] -Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) +(26) CometProject +Input [2]: [ca_address_sk#16, ca_county#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] -(32) Project [codegen id : 7] -Output [1]: [ca_address_sk#17] -Input [2]: [ca_address_sk#17, ca_county#18] +(27) CometBroadcastExchange +Input [1]: [ca_address_sk#16] +Arguments: [ca_address_sk#16] -(33) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ca_address_sk#16] +Arguments: [c_current_addr_sk#3], [ca_address_sk#16], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(29) CometProject +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16] +Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(35) Project [codegen id : 9] -Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] - -(36) Scan parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(30) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(31) CometFilter +Input [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Condition : isnotnull(cd_demo_sk#18) -(38) Filter [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#19) +(32) CometProject +Input [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#29, cd_purchase_estimate#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#23, 10, true, false, true) AS cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -(39) BroadcastExchange -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(33) CometBroadcastExchange +Input [9]: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -(40) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [1]: [c_current_cdemo_sk#2] +Right output [9]: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight -(41) Project [codegen id : 9] -Output [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(35) CometProject +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -(42) HashAggregate [codegen id : 9] -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(36) CometHashAggregate +Input [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -(43) Exchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(37) CometExchange +Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#31] +Arguments: hashpartitioning(cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(44) HashAggregate [codegen id : 10] -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(38) CometHashAggregate +Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#31] +Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, count(1)#30 AS cnt1#31, cd_purchase_estimate#23, count(1)#30 AS cnt2#32, cd_credit_rating#24, count(1)#30 AS cnt3#33, cd_dep_count#25, count(1)#30 AS cnt4#34, cd_dep_employed_count#26, count(1)#30 AS cnt5#35, cd_dep_college_count#27, count(1)#30 AS cnt6#36] -(45) TakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] -Arguments: 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] +(39) CometTakeOrderedAndProject +Input [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#27 ASC NULLS FIRST,cd_marital_status#28 ASC NULLS FIRST,cd_education_status#29 ASC NULLS FIRST,cd_purchase_estimate#22 ASC NULLS FIRST,cd_credit_rating#30 ASC NULLS FIRST,cd_dep_count#24 ASC NULLS FIRST,cd_dep_employed_count#25 ASC NULLS FIRST,cd_dep_college_count#26 ASC NULLS FIRST], output=[cd_gender#27,cd_marital_status#28,cd_education_status#29,cnt1#32,cd_purchase_estimate#22,cnt2#33,cd_credit_rating#30,cnt3#34,cd_dep_count#24,cnt4#35,cd_dep_employed_count#25,cnt5#36,cd_dep_college_count#26,cnt6#37]), [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37], 100, 0, [cd_gender#27 ASC NULLS FIRST, cd_marital_status#28 ASC NULLS FIRST, cd_education_status#29 ASC NULLS FIRST, cd_purchase_estimate#22 ASC NULLS FIRST, cd_credit_rating#30 ASC NULLS FIRST, cd_dep_count#24 ASC NULLS FIRST, cd_dep_employed_count#25 ASC NULLS FIRST, cd_dep_college_count#26 ASC NULLS FIRST], [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] + +(40) CometColumnarToRow [codegen id : 1] +Input [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/extended.txt new file mode 100644 index 0000000000..42c478c7c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/simplified.txt index 49aaf6d2b1..837c58b9aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/simplified.txt @@ -1,68 +1,42 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (10) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - Union - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] + CometHashAggregate [count] [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count(1)] + CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [customer_sk] #4 + CometUnion [customer_sk] + CometProject [ws_bill_customer_sk] [customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk] #5 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/explain.txt index d487982c71..4f4c84d308 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/explain.txt @@ -1,418 +1,383 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) -(6) Filter [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(7) BroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] - -(16) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(17) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(23) ReusedExchange [Reuses operator id: 7] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#20] -Right keys [1]: [ss_customer_sk#28] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(25) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#31] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] - -(32) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(33) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [5]: [c_customer_id#21 AS customer_id#36, c_first_name#22 AS customer_first_name#37, c_last_name#23 AS customer_last_name#38, c_email_address#27 AS customer_email_address#39, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#40] - -(35) BroadcastExchange -Input [5]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] + +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight + +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] + +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] +(35) CometFilter +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) -(39) Filter [codegen id : 10] -Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] -Condition : (isnotnull(c_customer_sk#41) AND isnotnull(c_customer_id#42)) +(36) CometProject +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] -(40) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] +(37) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#52)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] - -(42) Filter [codegen id : 8] -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Condition : isnotnull(ws_bill_customer_sk#49) - -(43) BroadcastExchange -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#41] -Right keys [1]: [ws_bill_customer_sk#49] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Input [12]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#53, d_year#54] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#52] -Right keys [1]: [d_date_sk#53] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#54] -Input [12]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52, d_date_sk#53, d_year#54] - -(49) HashAggregate [codegen id : 10] -Input [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#54] -Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] -Aggregate Attributes [1]: [sum#55] -Results [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] - -(50) Exchange -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] -Arguments: hashpartitioning(c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] -Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#57] -Results [2]: [c_customer_id#42 AS customer_id#58, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#57,18,2) AS year_total#59] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#58, year_total#59] -Condition : (isnotnull(year_total#59) AND (year_total#59 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#58, year_total#59] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#58] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [8]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, year_total#59] -Input [9]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, customer_id#58, year_total#59] - -(56) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] +(38) CometFilter +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_bill_customer_sk#58) + +(39) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight + +(41) CometProject +Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#62, d_year#63] + +(43) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Right output [2]: [d_date_sk#62, d_year#63] +Arguments: [ws_sold_date_sk#61], [d_date_sk#62], Inner, BuildRight + +(44) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#62, d_year#63] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] + +(45) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] + +(46) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] + +(48) CometFilter +Input [2]: [customer_id#65, year_total#66] +Condition : (isnotnull(year_total#66) AND (year_total#66 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#65, year_total#66] + +(50) CometBroadcastHashJoin +Left output [7]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Right output [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#22], [customer_id#65], Inner, BuildRight + +(51) CometProject +Input [9]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, customer_id#65, year_total#66] +Arguments: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66], [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] - -(58) Filter [codegen id : 14] -Input [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] -Condition : (isnotnull(c_customer_sk#60) AND isnotnull(c_customer_id#61)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [4]: [ws_bill_customer_sk#68, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#60] -Right keys [1]: [ws_bill_customer_sk#68] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] -Input [12]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_bill_customer_sk#68, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#72, d_year#73] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#71] -Right keys [1]: [d_date_sk#72] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, d_year#73] -Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71, d_date_sk#72, d_year#73] - -(65) HashAggregate [codegen id : 14] -Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, d_year#73] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))] -Aggregate Attributes [1]: [sum#74] -Results [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] - -(66) Exchange -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] -Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))#57] -Results [2]: [c_customer_id#61 AS customer_id#76, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))#57,18,2) AS year_total#77] - -(68) BroadcastExchange -Input [2]: [customer_id#76, year_total#77] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#76] -Join type: Inner -Join condition: (CASE WHEN (year_total#59 > 0.00) THEN (year_total#77 / year_total#59) ELSE 0E-20 END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#40 / year_total#19) ELSE 0E-20 END) - -(70) Project [codegen id : 16] -Output [4]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] -Input [10]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, year_total#59, customer_id#76, year_total#77] - -(71) TakeOrderedAndProject -Input [4]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] -Arguments: 100, [customer_id#36 ASC NULLS FIRST, customer_first_name#37 ASC NULLS FIRST, customer_last_name#38 ASC NULLS FIRST, customer_email_address#39 ASC NULLS FIRST], [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] +(53) CometFilter +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Condition : (isnotnull(c_customer_sk#67) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true))) + +(54) CometProject +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Arguments: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57], [c_customer_sk#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#69, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#70, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#71, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#73, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#74, 50, true, false, true) AS c_email_address#57] + +(55) ReusedExchange [Reuses operator id: 39] +Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] + +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_sk#67], [ws_bill_customer_sk#75], Inner, BuildRight + +(57) CometProject +Input [12]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#79, d_year#80] + +(59) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [ws_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(60) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] + +(61) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] + +(62) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#82, year_total#83] + +(65) CometBroadcastHashJoin +Left output [8]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] +Right output [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#22], [customer_id#82], Inner, (CASE WHEN (year_total#66 > 0.00) THEN (year_total#83 / year_total#66) ELSE 0E-20 END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#43 / year_total#23) ELSE 0E-20 END), BuildRight + +(66) CometProject +Input [10]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66, customer_id#82, year_total#83] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] + +(67) CometTakeOrderedAndProject +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_email_address#42 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_email_address#42]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], 100, 0, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_email_address#42 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] + +(68) CometColumnarToRow [codegen id : 1] +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/extended.txt new file mode 100644 index 0000000000..bcd6c3e73d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 75 out of 75 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/simplified.txt index 999922fcf3..21abb3ad03 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/simplified.txt @@ -1,106 +1,70 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] + CometProject [customer_id,customer_first_name,customer_last_name,customer_email_address] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #4 + CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/explain.txt index 0369b95bc7..39f1048822 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/explain.txt @@ -1,140 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16] -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/extended.txt new file mode 100644 index 0000000000..9b405bc9ee --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/simplified.txt index 52d223848e..33747406ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/explain.txt index 9035879b93..0a75495b92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/explain.txt @@ -1,724 +1,639 @@ == Physical Plan == -TakeOrderedAndProject (96) -+- * BroadcastHashJoin Inner BuildRight (95) - :- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - +- BroadcastExchange (94) - +- * Filter (93) - +- * HashAggregate (92) - +- Exchange (91) - +- * HashAggregate (90) - +- * Project (89) - +- * BroadcastHashJoin Inner BuildRight (88) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : :- * Filter (77) - : : : +- * ColumnarToRow (76) - : : : +- Scan parquet spark_catalog.default.store_sales (75) - : : +- ReusedExchange (78) - : +- ReusedExchange (80) - +- BroadcastExchange (87) - +- * Project (86) - +- * Filter (85) - +- * ColumnarToRow (84) - +- Scan parquet spark_catalog.default.date_dim (83) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometNativeScan parquet spark_catalog.default.date_dim (72) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [2]: [d_date_sk#39, d_week_seq#40] Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#43, isEmpty#44, count#45] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49, count(1)#50] -Results [6]: [store AS channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49 AS sales#52, count(1)#50 AS number_sales#53] -(74) Filter [codegen id : 52] -Input [6]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53] -Condition : (isnotnull(sales#52) AND (cast(sales#52 as decimal(32,6)) > cast(Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] +(65) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#59)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] - -(77) Filter [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) -(78) ReusedExchange [Reuses operator id: 54] +(67) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(79) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#51], [ss_item_sk#34], LeftSemi, BuildRight -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] -(81) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [i_item_sk#60] -Join type: Inner -Join condition: None +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_item_sk#51], [i_item_sk#55], Inner, BuildRight -(82) Project [codegen id : 50] -Output [6]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [8]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] -(83) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#59, d_week_seq#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(85) Filter [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(86) Project [codegen id : 49] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(87) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(88) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_sold_date_sk#59] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 50] -Output [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [7]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] - -(90) HashAggregate [codegen id : 50] -Input [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), partial_count(1)] -Aggregate Attributes [3]: [sum#68, isEmpty#69, count#70] -Results [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] - -(91) Exchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(92) HashAggregate [codegen id : 51] -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74, count(1)#75] -Results [6]: [store AS channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74 AS sales#77, count(1)#75 AS number_sales#78] - -(93) Filter [codegen id : 51] -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(94) BroadcastExchange -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 52] -Left keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Right keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Join type: Inner -Join condition: None - -(96) TakeOrderedAndProject -Input [12]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] +(73) CometFilter +Input [2]: [d_date_sk#59, d_week_seq#60] +Condition : ((isnotnull(d_week_seq#60) AND (d_week_seq#60 = Subquery scalar-subquery#61, [id=#62])) AND isnotnull(d_date_sk#59)) + +(74) CometProject +Input [2]: [d_date_sk#59, d_week_seq#60] +Arguments: [d_date_sk#59], [d_date_sk#59] + +(75) CometBroadcastExchange +Input [1]: [d_date_sk#59] +Arguments: [d_date_sk#59] + +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [ss_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight + +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] + +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] + +(79) CometExchange +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(80) CometHashAggregate +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] + +(81) CometFilter +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Condition : (isnotnull(sales#67) AND (cast(sales#67 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(82) CometBroadcastExchange +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#56, i_class_id#57, i_category_id#58], Inner, BuildRight + +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#66,i_brand_id#56,i_class_id#57,i_category_id#58,sales#67,number_sales#68]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#54, [id=#55] -* HashAggregate (115) -+- Exchange (114) - +- * HashAggregate (113) - +- Union (112) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * ColumnarToRow (98) - : : +- Scan parquet spark_catalog.default.store_sales (97) - : +- ReusedExchange (99) - :- * Project (106) - : +- * BroadcastHashJoin Inner BuildRight (105) - : :- * ColumnarToRow (103) - : : +- Scan parquet spark_catalog.default.catalog_sales (102) - : +- ReusedExchange (104) - +- * Project (111) - +- * BroadcastHashJoin Inner BuildRight (110) - :- * ColumnarToRow (108) - : +- Scan parquet spark_catalog.default.web_sales (107) - +- ReusedExchange (109) - - -(97) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometNativeScan parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometNativeScan parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#81)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#71)] ReadSchema: struct -(98) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +(87) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#72] -(99) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#82] +(88) CometBroadcastHashJoin +Left output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] +Right output [1]: [d_date_sk#72] +Arguments: [ss_sold_date_sk#71], [d_date_sk#72], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#81] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None +(89) CometProject +Input [4]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71, d_date_sk#72] +Arguments: [quantity#73, list_price#74], [ss_quantity#69 AS quantity#73, ss_list_price#70 AS list_price#74] -(101) Project [codegen id : 2] -Output [2]: [ss_quantity#79 AS quantity#83, ss_list_price#80 AS list_price#84] -Input [4]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81, d_date_sk#82] - -(102) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] +(90) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#87)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] - -(104) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#88] +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#78] -(105) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#87] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] +Right output [1]: [d_date_sk#78] +Arguments: [cs_sold_date_sk#77], [d_date_sk#78], Inner, BuildRight -(106) Project [codegen id : 4] -Output [2]: [cs_quantity#85 AS quantity#89, cs_list_price#86 AS list_price#90] -Input [4]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87, d_date_sk#88] +(93) CometProject +Input [4]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#78] +Arguments: [quantity#79, list_price#80], [cs_quantity#75 AS quantity#79, cs_list_price#76 AS list_price#80] -(107) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(94) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#93)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#83)] ReadSchema: struct -(108) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#84] -(109) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#94] +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] +Right output [1]: [d_date_sk#84] +Arguments: [ws_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight -(110) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#93] -Right keys [1]: [d_date_sk#94] -Join type: Inner -Join condition: None +(97) CometProject +Input [4]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83, d_date_sk#84] +Arguments: [quantity#85, list_price#86], [ws_quantity#81 AS quantity#85, ws_list_price#82 AS list_price#86] -(111) Project [codegen id : 6] -Output [2]: [ws_quantity#91 AS quantity#95, ws_list_price#92 AS list_price#96] -Input [4]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93, d_date_sk#94] +(98) CometUnion +Child 0 Input [2]: [quantity#73, list_price#74] +Child 1 Input [2]: [quantity#79, list_price#80] +Child 2 Input [2]: [quantity#85, list_price#86] -(112) Union - -(113) HashAggregate [codegen id : 7] -Input [2]: [quantity#83, list_price#84] +(99) CometHashAggregate +Input [2]: [quantity#73, list_price#74] Keys: [] -Functions [1]: [partial_avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [2]: [sum#97, count#98] -Results [2]: [sum#99, count#100] +Functions [1]: [partial_avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] -(114) Exchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(100) CometExchange +Input [2]: [sum#87, count#88] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(115) HashAggregate [codegen id : 8] -Input [2]: [sum#99, count#100] +(101) CometHashAggregate +Input [2]: [sum#87, count#88] Keys: [] -Functions [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101] -Results [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101 AS average_sales#102] +Functions [1]: [avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] + +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#89] -Subquery:2 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* Project (119) -+- * Filter (118) - +- * ColumnarToRow (117) - +- Scan parquet spark_catalog.default.date_dim (116) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometNativeScan parquet spark_catalog.default.date_dim (103) -(116) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(103) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(117) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(104) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1999)) AND (d_moy#92 = 12)) AND (d_dom#93 = 16)) -(118) Filter [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Condition : (((((isnotnull(d_year#104) AND isnotnull(d_moy#105)) AND isnotnull(d_dom#106)) AND (d_year#104 = 1999)) AND (d_moy#105 = 12)) AND (d_dom#106 = 16)) +(105) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(119) Project [codegen id : 1] -Output [1]: [d_week_seq#103] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] -Subquery:3 Hosting operator id = 93 Hosting Expression = ReusedSubquery Subquery scalar-subquery#54, [id=#55] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:4 Hosting operator id = 85 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* Project (123) -+- * Filter (122) - +- * ColumnarToRow (121) - +- Scan parquet spark_catalog.default.date_dim (120) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#61, [id=#62] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometNativeScan parquet spark_catalog.default.date_dim (107) -(120) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(107) CometNativeScan parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(121) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(108) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1998)) AND (d_moy#92 = 12)) AND (d_dom#93 = 16)) -(122) Filter [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Condition : (((((isnotnull(d_year#108) AND isnotnull(d_moy#109)) AND isnotnull(d_dom#110)) AND (d_year#108 = 1998)) AND (d_moy#109 = 12)) AND (d_dom#110 = 16)) +(109) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(123) Project [codegen id : 1] -Output [1]: [d_week_seq#107] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/extended.txt new file mode 100644 index 0000000000..fc08251119 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/extended.txt @@ -0,0 +1,290 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometBroadcastHashJoin + :- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + : +- Subquery + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 280 out of 284 eligible operators (98%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/simplified.txt index b9a845cb11..92429c0948 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/simplified.txt @@ -1,191 +1,122 @@ -TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - WholeStageCodegen (52) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #2 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #1 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #2 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #3 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #4 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #1 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #4 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (51) - Filter [sales] + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (50) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (49) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #3 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/explain.txt index 5f5d49ab21..c7b1d65cbf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/explain.txt @@ -1,931 +1,812 @@ == Physical Plan == -TakeOrderedAndProject (133) -+- * HashAggregate (132) - +- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * HashAggregate (108) - : +- Exchange (107) - : +- * HashAggregate (106) - : +- Union (105) - : :- * Filter (74) - : : +- * HashAggregate (73) - : : +- Exchange (72) - : : +- * HashAggregate (71) - : : +- * Project (70) - : : +- * BroadcastHashJoin Inner BuildRight (69) - : : :- * Project (63) - : : : +- * BroadcastHashJoin Inner BuildRight (62) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (54) - : : : : +- * Project (53) - : : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : : :- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.item (4) - : : : : +- BroadcastExchange (51) - : : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : : :- * HashAggregate (39) - : : : : : +- Exchange (38) - : : : : : +- * HashAggregate (37) - : : : : : +- * Project (36) - : : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : : :- * Project (33) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : : :- * Filter (9) - : : : : : : : +- * ColumnarToRow (8) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : : +- BroadcastExchange (31) - : : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : : :- * Filter (12) - : : : : : : : +- * ColumnarToRow (11) - : : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : : +- BroadcastExchange (29) - : : : : : : +- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Filter (15) - : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : : +- ReusedExchange (34) - : : : : +- BroadcastExchange (49) - : : : : +- * Project (48) - : : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : : :- * Project (45) - : : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : : :- * Filter (42) - : : : : : : +- * ColumnarToRow (41) - : : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : : +- ReusedExchange (43) - : : : : +- ReusedExchange (46) - : : : +- BroadcastExchange (61) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : : :- * Filter (58) - : : : : +- * ColumnarToRow (57) - : : : : +- Scan parquet spark_catalog.default.item (56) - : : : +- ReusedExchange (59) - : : +- BroadcastExchange (68) - : : +- * Project (67) - : : +- * Filter (66) - : : +- * ColumnarToRow (65) - : : +- Scan parquet spark_catalog.default.date_dim (64) - : :- * Filter (89) - : : +- * HashAggregate (88) - : : +- Exchange (87) - : : +- * HashAggregate (86) - : : +- * Project (85) - : : +- * BroadcastHashJoin Inner BuildRight (84) - : : :- * Project (82) - : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : : : :- * Filter (77) - : : : : : +- * ColumnarToRow (76) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : : +- ReusedExchange (78) - : : : +- ReusedExchange (80) - : : +- ReusedExchange (83) - : +- * Filter (104) - : +- * HashAggregate (103) - : +- Exchange (102) - : +- * HashAggregate (101) - : +- * Project (100) - : +- * BroadcastHashJoin Inner BuildRight (99) - : :- * Project (97) - : : +- * BroadcastHashJoin Inner BuildRight (96) - : : :- * BroadcastHashJoin LeftSemi BuildRight (94) - : : : :- * Filter (92) - : : : : +- * ColumnarToRow (91) - : : : : +- Scan parquet spark_catalog.default.web_sales (90) - : : : +- ReusedExchange (93) - : : +- ReusedExchange (95) - : +- ReusedExchange (98) - :- * HashAggregate (113) - : +- Exchange (112) - : +- * HashAggregate (111) - : +- * HashAggregate (110) - : +- ReusedExchange (109) - :- * HashAggregate (118) - : +- Exchange (117) - : +- * HashAggregate (116) - : +- * HashAggregate (115) - : +- ReusedExchange (114) - :- * HashAggregate (123) - : +- Exchange (122) - : +- * HashAggregate (121) - : +- * HashAggregate (120) - : +- ReusedExchange (119) - +- * HashAggregate (128) - +- Exchange (127) - +- * HashAggregate (126) - +- * HashAggregate (125) - +- ReusedExchange (124) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (122) ++- CometTakeOrderedAndProject (121) + +- CometHashAggregate (120) + +- CometExchange (119) + +- CometHashAggregate (118) + +- CometUnion (117) + :- CometHashAggregate (96) + : +- CometExchange (95) + : +- CometHashAggregate (94) + : +- CometUnion (93) + : :- CometFilter (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometBroadcastHashJoin (47) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (46) + : : : : +- CometProject (45) + : : : : +- CometBroadcastHashJoin (44) + : : : : :- CometFilter (4) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : : +- CometBroadcastExchange (43) + : : : : +- CometBroadcastHashJoin (42) + : : : : :- CometHashAggregate (32) + : : : : : +- CometExchange (31) + : : : : : +- CometHashAggregate (30) + : : : : : +- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (26) + : : : : : : +- CometBroadcastHashJoin (25) + : : : : : : :- CometFilter (6) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometBroadcastExchange (24) + : : : : : : +- CometBroadcastHashJoin (23) + : : : : : : :- CometFilter (8) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (7) + : : : : : : +- CometBroadcastExchange (22) + : : : : : : +- CometProject (21) + : : : : : : +- CometBroadcastHashJoin (20) + : : : : : : :- CometProject (15) + : : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : : :- CometFilter (10) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometBroadcastExchange (13) + : : : : : : : +- CometFilter (12) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (11) + : : : : : : +- CometBroadcastExchange (19) + : : : : : : +- CometProject (18) + : : : : : : +- CometFilter (17) + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (16) + : : : : : +- ReusedExchange (27) + : : : : +- CometBroadcastExchange (41) + : : : : +- CometProject (40) + : : : : +- CometBroadcastHashJoin (39) + : : : : :- CometProject (37) + : : : : : +- CometBroadcastHashJoin (36) + : : : : : :- CometFilter (34) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (33) + : : : : : +- ReusedExchange (35) + : : : : +- ReusedExchange (38) + : : : +- CometBroadcastExchange (52) + : : : +- CometBroadcastHashJoin (51) + : : : :- CometFilter (49) + : : : : +- CometNativeScan parquet spark_catalog.default.item (48) + : : : +- ReusedExchange (50) + : : +- CometBroadcastExchange (58) + : : +- CometProject (57) + : : +- CometFilter (56) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (55) + : :- CometFilter (78) + : : +- CometHashAggregate (77) + : : +- CometExchange (76) + : : +- CometHashAggregate (75) + : : +- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometBroadcastHashJoin (68) + : : : : :- CometFilter (66) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (65) + : : : : +- ReusedExchange (67) + : : : +- ReusedExchange (69) + : : +- ReusedExchange (72) + : +- CometFilter (92) + : +- CometHashAggregate (91) + : +- CometExchange (90) + : +- CometHashAggregate (89) + : +- CometProject (88) + : +- CometBroadcastHashJoin (87) + : :- CometProject (85) + : : +- CometBroadcastHashJoin (84) + : : :- CometBroadcastHashJoin (82) + : : : :- CometFilter (80) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (79) + : : : +- ReusedExchange (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (86) + :- CometHashAggregate (101) + : +- CometExchange (100) + : +- CometHashAggregate (99) + : +- CometHashAggregate (98) + : +- ReusedExchange (97) + :- CometHashAggregate (106) + : +- CometExchange (105) + : +- CometHashAggregate (104) + : +- CometHashAggregate (103) + : +- ReusedExchange (102) + :- CometHashAggregate (111) + : +- CometExchange (110) + : +- CometHashAggregate (109) + : +- CometHashAggregate (108) + : +- ReusedExchange (107) + +- CometHashAggregate (116) + +- CometExchange (115) + +- CometHashAggregate (114) + +- CometHashAggregate (113) + +- ReusedExchange (112) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometNativeScan parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometNativeScan parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : isnotnull(i_item_sk#35) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [3]: [d_date_sk#39, d_year#40, d_moy#41] Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#42, isEmpty#43, count#44] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 26] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48, count(1)#49] -Results [6]: [store AS channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48 AS sales#51, count(1)#49 AS number_sales#52] -(74) Filter [codegen id : 26] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) -(75) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] +(65) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 51] -Input [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] - -(77) Filter [codegen id : 51] -Input [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] -Condition : isnotnull(cs_item_sk#55) +(66) CometFilter +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) -(78) ReusedExchange [Reuses operator id: 54] +(67) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(79) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#55] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] - -(81) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#55] -Right keys [1]: [i_item_sk#59] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 51] -Output [6]: [cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_brand_id#60, i_class_id#61, i_category_id#62] -Input [8]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] - -(83) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#63] - -(84) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#63] -Join type: Inner -Join condition: None - -(85) Project [codegen id : 51] -Output [5]: [cs_quantity#56, cs_list_price#57, i_brand_id#60, i_class_id#61, i_category_id#62] -Input [7]: [cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_brand_id#60, i_class_id#61, i_category_id#62, d_date_sk#63] - -(86) HashAggregate [codegen id : 51] -Input [5]: [cs_quantity#56, cs_list_price#57, i_brand_id#60, i_class_id#61, i_category_id#62] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [partial_sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57)), partial_count(1)] -Aggregate Attributes [3]: [sum#64, isEmpty#65, count#66] -Results [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] - -(87) Exchange -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] -Arguments: hashpartitioning(i_brand_id#60, i_class_id#61, i_category_id#62, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(88) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57))#70, count(1)#71] -Results [6]: [catalog AS channel#72, i_brand_id#60, i_class_id#61, i_category_id#62, sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57))#70 AS sales#73, count(1)#71 AS number_sales#74] - -(89) Filter [codegen id : 52] -Input [6]: [channel#72, i_brand_id#60, i_class_id#61, i_category_id#62, sales#73, number_sales#74] -Condition : (isnotnull(sales#73) AND (cast(sales#73 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) - -(90) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] +(68) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#34] +Arguments: [cs_item_sk#50], [ss_item_sk#34], LeftSemi, BuildRight + +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] + +(70) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] +Arguments: [cs_item_sk#50], [i_item_sk#54], Inner, BuildRight + +(71) CometProject +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57] + +(72) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#58] + +(73) CometBroadcastHashJoin +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57] +Right output [1]: [d_date_sk#58] +Arguments: [cs_sold_date_sk#53], [d_date_sk#58], Inner, BuildRight + +(74) CometProject +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57, d_date_sk#58] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57], [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57] + +(75) CometHashAggregate +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57] +Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] + +(76) CometExchange +Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#59, isEmpty#60, count#61] +Arguments: hashpartitioning(i_brand_id#55, i_class_id#56, i_category_id#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(77) CometHashAggregate +Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#59, isEmpty#60, count#61] +Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] + +(78) CometFilter +Input [6]: [channel#62, i_brand_id#55, i_class_id#56, i_category_id#57, sales#63, number_sales#64] +Condition : (isnotnull(sales#63) AND (cast(sales#63 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(79) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#78)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#68)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(91) ColumnarToRow [codegen id : 77] -Input [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] - -(92) Filter [codegen id : 77] -Input [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] -Condition : isnotnull(ws_item_sk#75) +(80) CometFilter +Input [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_item_sk#65) -(93) ReusedExchange [Reuses operator id: 54] +(81) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(94) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(95) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#79, i_brand_id#80, i_class_id#81, i_category_id#82] - -(96) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [i_item_sk#79] -Join type: Inner -Join condition: None - -(97) Project [codegen id : 77] -Output [6]: [ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_brand_id#80, i_class_id#81, i_category_id#82] -Input [8]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_item_sk#79, i_brand_id#80, i_class_id#81, i_category_id#82] - -(98) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#83] - -(99) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_sold_date_sk#78] -Right keys [1]: [d_date_sk#83] -Join type: Inner -Join condition: None - -(100) Project [codegen id : 77] -Output [5]: [ws_quantity#76, ws_list_price#77, i_brand_id#80, i_class_id#81, i_category_id#82] -Input [7]: [ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_brand_id#80, i_class_id#81, i_category_id#82, d_date_sk#83] - -(101) HashAggregate [codegen id : 77] -Input [5]: [ws_quantity#76, ws_list_price#77, i_brand_id#80, i_class_id#81, i_category_id#82] -Keys [3]: [i_brand_id#80, i_class_id#81, i_category_id#82] -Functions [2]: [partial_sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77)), partial_count(1)] -Aggregate Attributes [3]: [sum#84, isEmpty#85, count#86] -Results [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] - -(102) Exchange -Input [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] -Arguments: hashpartitioning(i_brand_id#80, i_class_id#81, i_category_id#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(103) HashAggregate [codegen id : 78] -Input [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] -Keys [3]: [i_brand_id#80, i_class_id#81, i_category_id#82] -Functions [2]: [sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77))#90, count(1)#91] -Results [6]: [web AS channel#92, i_brand_id#80, i_class_id#81, i_category_id#82, sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77))#90 AS sales#93, count(1)#91 AS number_sales#94] - -(104) Filter [codegen id : 78] -Input [6]: [channel#92, i_brand_id#80, i_class_id#81, i_category_id#82, sales#93, number_sales#94] -Condition : (isnotnull(sales#93) AND (cast(sales#93 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) - -(105) Union - -(106) HashAggregate [codegen id : 79] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sales#51, number_sales#52] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [partial_sum(sales#51), partial_sum(number_sales#52)] -Aggregate Attributes [3]: [sum#95, isEmpty#96, sum#97] -Results [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(107) Exchange -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(108) HashAggregate [codegen id : 80] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(109) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(110) HashAggregate [codegen id : 160] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [5]: [channel#50, i_brand_id#36, i_class_id#37, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(111) HashAggregate [codegen id : 160] -Input [5]: [channel#50, i_brand_id#36, i_class_id#37, sum_sales#103, number_sales#104] -Keys [3]: [channel#50, i_brand_id#36, i_class_id#37] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#105, isEmpty#106, sum#107] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] - -(112) Exchange -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(113) HashAggregate [codegen id : 161] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] -Keys [3]: [channel#50, i_brand_id#36, i_class_id#37] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#111, sum(number_sales#104)#112] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, null AS i_category_id#113, sum(sum_sales#103)#111 AS sum(sum_sales)#114, sum(number_sales#104)#112 AS sum(number_sales)#115] - -(114) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(115) HashAggregate [codegen id : 241] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [4]: [channel#50, i_brand_id#36, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(116) HashAggregate [codegen id : 241] -Input [4]: [channel#50, i_brand_id#36, sum_sales#103, number_sales#104] -Keys [2]: [channel#50, i_brand_id#36] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#116, isEmpty#117, sum#118] -Results [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] - -(117) Exchange -Input [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] -Arguments: hashpartitioning(channel#50, i_brand_id#36, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(118) HashAggregate [codegen id : 242] -Input [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] -Keys [2]: [channel#50, i_brand_id#36] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#122, sum(number_sales#104)#123] -Results [6]: [channel#50, i_brand_id#36, null AS i_class_id#124, null AS i_category_id#125, sum(sum_sales#103)#122 AS sum(sum_sales)#126, sum(number_sales#104)#123 AS sum(number_sales)#127] - -(119) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(120) HashAggregate [codegen id : 322] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [3]: [channel#50, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(121) HashAggregate [codegen id : 322] -Input [3]: [channel#50, sum_sales#103, number_sales#104] -Keys [1]: [channel#50] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#128, isEmpty#129, sum#130] -Results [4]: [channel#50, sum#131, isEmpty#132, sum#133] - -(122) Exchange -Input [4]: [channel#50, sum#131, isEmpty#132, sum#133] -Arguments: hashpartitioning(channel#50, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(123) HashAggregate [codegen id : 323] -Input [4]: [channel#50, sum#131, isEmpty#132, sum#133] -Keys [1]: [channel#50] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#134, sum(number_sales#104)#135] -Results [6]: [channel#50, null AS i_brand_id#136, null AS i_class_id#137, null AS i_category_id#138, sum(sum_sales#103)#134 AS sum(sum_sales)#139, sum(number_sales#104)#135 AS sum(number_sales)#140] - -(124) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(125) HashAggregate [codegen id : 403] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [2]: [sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(126) HashAggregate [codegen id : 403] -Input [2]: [sum_sales#103, number_sales#104] +(82) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Right output [1]: [ss_item_sk#34] +Arguments: [ws_item_sk#65], [ss_item_sk#34], LeftSemi, BuildRight + +(83) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Right output [4]: [i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] +Arguments: [ws_item_sk#65], [i_item_sk#69], Inner, BuildRight + +(85) CometProject +Input [8]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] +Arguments: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72], [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72] + +(86) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#73] + +(87) CometBroadcastHashJoin +Left output [6]: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72] +Right output [1]: [d_date_sk#73] +Arguments: [ws_sold_date_sk#68], [d_date_sk#73], Inner, BuildRight + +(88) CometProject +Input [7]: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72, d_date_sk#73] +Arguments: [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72], [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72] + +(89) CometHashAggregate +Input [5]: [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72] +Keys [3]: [i_brand_id#70, i_class_id#71, i_category_id#72] +Functions [2]: [partial_sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), partial_count(1)] + +(90) CometExchange +Input [6]: [i_brand_id#70, i_class_id#71, i_category_id#72, sum#74, isEmpty#75, count#76] +Arguments: hashpartitioning(i_brand_id#70, i_class_id#71, i_category_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(91) CometHashAggregate +Input [6]: [i_brand_id#70, i_class_id#71, i_category_id#72, sum#74, isEmpty#75, count#76] +Keys [3]: [i_brand_id#70, i_class_id#71, i_category_id#72] +Functions [2]: [sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), count(1)] + +(92) CometFilter +Input [6]: [channel#77, i_brand_id#70, i_class_id#71, i_category_id#72, sales#78, number_sales#79] +Condition : (isnotnull(sales#78) AND (cast(sales#78 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(93) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#62, i_brand_id#55, i_class_id#56, i_category_id#57, sales#63, number_sales#64] +Child 2 Input [6]: [channel#77, i_brand_id#70, i_class_id#71, i_category_id#72, sales#78, number_sales#79] + +(94) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] + +(95) CometExchange +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(96) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(97) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(98) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(99) CometHashAggregate +Input [5]: [channel#45, i_brand_id#36, i_class_id#37, sum_sales#83, number_sales#84] +Keys [3]: [channel#45, i_brand_id#36, i_class_id#37] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] + +(100) CometExchange +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, sum#85, isEmpty#86, sum#87] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(101) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, sum#85, isEmpty#86, sum#87] +Keys [3]: [channel#45, i_brand_id#36, i_class_id#37] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(102) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(103) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(104) CometHashAggregate +Input [4]: [channel#45, i_brand_id#36, sum_sales#83, number_sales#84] +Keys [2]: [channel#45, i_brand_id#36] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] + +(105) CometExchange +Input [5]: [channel#45, i_brand_id#36, sum#88, isEmpty#89, sum#90] +Arguments: hashpartitioning(channel#45, i_brand_id#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(106) CometHashAggregate +Input [5]: [channel#45, i_brand_id#36, sum#88, isEmpty#89, sum#90] +Keys [2]: [channel#45, i_brand_id#36] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(107) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(108) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(109) CometHashAggregate +Input [3]: [channel#45, sum_sales#83, number_sales#84] +Keys [1]: [channel#45] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] + +(110) CometExchange +Input [4]: [channel#45, sum#91, isEmpty#92, sum#93] +Arguments: hashpartitioning(channel#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(111) CometHashAggregate +Input [4]: [channel#45, sum#91, isEmpty#92, sum#93] +Keys [1]: [channel#45] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(112) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(113) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(114) CometHashAggregate +Input [2]: [sum_sales#83, number_sales#84] Keys: [] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#141, isEmpty#142, sum#143] -Results [3]: [sum#144, isEmpty#145, sum#146] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] -(127) Exchange -Input [3]: [sum#144, isEmpty#145, sum#146] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] +(115) CometExchange +Input [3]: [sum#94, isEmpty#95, sum#96] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(128) HashAggregate [codegen id : 404] -Input [3]: [sum#144, isEmpty#145, sum#146] +(116) CometHashAggregate +Input [3]: [sum#94, isEmpty#95, sum#96] Keys: [] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#147, sum(number_sales#104)#148] -Results [6]: [null AS channel#149, null AS i_brand_id#150, null AS i_class_id#151, null AS i_category_id#152, sum(sum_sales#103)#147 AS sum(sum_sales)#153, sum(number_sales#104)#148 AS sum(number_sales)#154] - -(129) Union - -(130) HashAggregate [codegen id : 405] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Keys [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(117) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Child 1 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#97, sum(sum_sales)#98, sum(number_sales)#99] +Child 2 Input [6]: [channel#45, i_brand_id#36, i_class_id#100, i_category_id#101, sum(sum_sales)#102, sum(number_sales)#103] +Child 3 Input [6]: [channel#45, i_brand_id#104, i_class_id#105, i_category_id#106, sum(sum_sales)#107, sum(number_sales)#108] +Child 4 Input [6]: [channel#109, i_brand_id#110, i_class_id#111, i_category_id#112, sum(sum_sales)#113, sum(number_sales)#114] + +(118) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] Functions: [] -Aggregate Attributes: [] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -(131) Exchange -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104, 5), ENSURE_REQUIREMENTS, [plan_id=19] +(119) CometExchange +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(132) HashAggregate [codegen id : 406] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Keys [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] +(120) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] Functions: [] -Aggregate Attributes: [] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -(133) TakeOrderedAndProject -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Arguments: 100, [channel#50 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] +(121) CometTakeOrderedAndProject +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#83,number_sales#84]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] + +(122) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#53, [id=#54] -* HashAggregate (156) -+- Exchange (155) - +- * HashAggregate (154) - +- Union (153) - :- * Project (138) - : +- * BroadcastHashJoin Inner BuildRight (137) - : :- * ColumnarToRow (135) - : : +- Scan parquet spark_catalog.default.store_sales (134) - : +- ReusedExchange (136) - :- * Project (147) - : +- * BroadcastHashJoin Inner BuildRight (146) - : :- * ColumnarToRow (140) - : : +- Scan parquet spark_catalog.default.catalog_sales (139) - : +- BroadcastExchange (145) - : +- * Project (144) - : +- * Filter (143) - : +- * ColumnarToRow (142) - : +- Scan parquet spark_catalog.default.date_dim (141) - +- * Project (152) - +- * BroadcastHashJoin Inner BuildRight (151) - :- * ColumnarToRow (149) - : +- Scan parquet spark_catalog.default.web_sales (148) - +- ReusedExchange (150) - - -(134) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] +* CometColumnarToRow (142) ++- CometHashAggregate (141) + +- CometExchange (140) + +- CometHashAggregate (139) + +- CometUnion (138) + :- CometProject (126) + : +- CometBroadcastHashJoin (125) + : :- CometNativeScan parquet spark_catalog.default.store_sales (123) + : +- ReusedExchange (124) + :- CometProject (133) + : +- CometBroadcastHashJoin (132) + : :- CometNativeScan parquet spark_catalog.default.catalog_sales (127) + : +- CometBroadcastExchange (131) + : +- CometProject (130) + : +- CometFilter (129) + : +- CometNativeScan parquet spark_catalog.default.date_dim (128) + +- CometProject (137) + +- CometBroadcastHashJoin (136) + :- CometNativeScan parquet spark_catalog.default.web_sales (134) + +- ReusedExchange (135) + + +(123) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#157)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#117)] ReadSchema: struct -(135) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157] +(124) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#118] -(136) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#158] +(125) CometBroadcastHashJoin +Left output [3]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117] +Right output [1]: [d_date_sk#118] +Arguments: [ss_sold_date_sk#117], [d_date_sk#118], Inner, BuildRight -(137) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#157] -Right keys [1]: [d_date_sk#158] -Join type: Inner -Join condition: None +(126) CometProject +Input [4]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117, d_date_sk#118] +Arguments: [quantity#119, list_price#120], [ss_quantity#115 AS quantity#119, ss_list_price#116 AS list_price#120] -(138) Project [codegen id : 2] -Output [2]: [ss_quantity#155 AS quantity#159, ss_list_price#156 AS list_price#160] -Input [4]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157, d_date_sk#158] - -(139) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163] +(127) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#163)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#123)] ReadSchema: struct -(140) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163] - -(141) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#164, d_year#165] +(128) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#124, d_year#125] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(142) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#164, d_year#165] - -(143) Filter [codegen id : 3] -Input [2]: [d_date_sk#164, d_year#165] -Condition : (((isnotnull(d_year#165) AND (d_year#165 >= 1998)) AND (d_year#165 <= 2000)) AND isnotnull(d_date_sk#164)) +(129) CometFilter +Input [2]: [d_date_sk#124, d_year#125] +Condition : (((isnotnull(d_year#125) AND (d_year#125 >= 1998)) AND (d_year#125 <= 2000)) AND isnotnull(d_date_sk#124)) -(144) Project [codegen id : 3] -Output [1]: [d_date_sk#164] -Input [2]: [d_date_sk#164, d_year#165] +(130) CometProject +Input [2]: [d_date_sk#124, d_year#125] +Arguments: [d_date_sk#124], [d_date_sk#124] -(145) BroadcastExchange -Input [1]: [d_date_sk#164] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] +(131) CometBroadcastExchange +Input [1]: [d_date_sk#124] +Arguments: [d_date_sk#124] -(146) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#163] -Right keys [1]: [d_date_sk#164] -Join type: Inner -Join condition: None +(132) CometBroadcastHashJoin +Left output [3]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123] +Right output [1]: [d_date_sk#124] +Arguments: [cs_sold_date_sk#123], [d_date_sk#124], Inner, BuildRight -(147) Project [codegen id : 4] -Output [2]: [cs_quantity#161 AS quantity#166, cs_list_price#162 AS list_price#167] -Input [4]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163, d_date_sk#164] +(133) CometProject +Input [4]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123, d_date_sk#124] +Arguments: [quantity#126, list_price#127], [cs_quantity#121 AS quantity#126, cs_list_price#122 AS list_price#127] -(148) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170] +(134) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#170)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#130)] ReadSchema: struct -(149) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170] +(135) ReusedExchange [Reuses operator id: 131] +Output [1]: [d_date_sk#131] -(150) ReusedExchange [Reuses operator id: 145] -Output [1]: [d_date_sk#171] +(136) CometBroadcastHashJoin +Left output [3]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130] +Right output [1]: [d_date_sk#131] +Arguments: [ws_sold_date_sk#130], [d_date_sk#131], Inner, BuildRight -(151) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#170] -Right keys [1]: [d_date_sk#171] -Join type: Inner -Join condition: None +(137) CometProject +Input [4]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130, d_date_sk#131] +Arguments: [quantity#132, list_price#133], [ws_quantity#128 AS quantity#132, ws_list_price#129 AS list_price#133] -(152) Project [codegen id : 6] -Output [2]: [ws_quantity#168 AS quantity#172, ws_list_price#169 AS list_price#173] -Input [4]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170, d_date_sk#171] +(138) CometUnion +Child 0 Input [2]: [quantity#119, list_price#120] +Child 1 Input [2]: [quantity#126, list_price#127] +Child 2 Input [2]: [quantity#132, list_price#133] -(153) Union - -(154) HashAggregate [codegen id : 7] -Input [2]: [quantity#159, list_price#160] +(139) CometHashAggregate +Input [2]: [quantity#119, list_price#120] Keys: [] -Functions [1]: [partial_avg((cast(quantity#159 as decimal(10,0)) * list_price#160))] -Aggregate Attributes [2]: [sum#174, count#175] -Results [2]: [sum#176, count#177] +Functions [1]: [partial_avg((cast(quantity#119 as decimal(10,0)) * list_price#120))] -(155) Exchange -Input [2]: [sum#176, count#177] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21] +(140) CometExchange +Input [2]: [sum#134, count#135] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(156) HashAggregate [codegen id : 8] -Input [2]: [sum#176, count#177] +(141) CometHashAggregate +Input [2]: [sum#134, count#135] Keys: [] -Functions [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))] -Aggregate Attributes [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))#178] -Results [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))#178 AS average_sales#179] +Functions [1]: [avg((cast(quantity#119 as decimal(10,0)) * list_price#120))] + +(142) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#136] -Subquery:2 Hosting operator id = 89 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] -Subquery:3 Hosting operator id = 104 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/extended.txt new file mode 100644 index 0000000000..c24fc36bd9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/extended.txt @@ -0,0 +1,2030 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometFilter + : : : +- Subquery + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometUnion + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : :- CometFilter + : : : +- ReusedSubquery + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometHashAggregate + : : : : : +- CometExchange + : : : : : +- CometHashAggregate + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- CometFilter + : : +- ReusedSubquery + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometHashAggregate + : : : : +- CometExchange + : : : : +- CometHashAggregate + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometFilter + : +- ReusedSubquery + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 2007 out of 2022 eligible operators (99%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/simplified.txt index be14bd5141..ff1ccebc20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/simplified.txt @@ -1,249 +1,149 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - WholeStageCodegen (406) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 - WholeStageCodegen (405) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - InputAdapter - Union - WholeStageCodegen (80) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (79) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - InputAdapter - Union - WholeStageCodegen (26) - Filter [sales] - Subquery #1 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #14 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #10 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #15 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #3 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #6 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #10 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - InputAdapter - ReusedExchange [d_date_sk] #10 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (52) - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #16 - WholeStageCodegen (51) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - InputAdapter - ReusedExchange [d_date_sk] #13 - WholeStageCodegen (78) - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #17 - WholeStageCodegen (77) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - InputAdapter - ReusedExchange [d_date_sk] #13 - WholeStageCodegen (161) - HashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id] #18 - WholeStageCodegen (160) - HashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (242) - HashAggregate [channel,i_brand_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id] #19 - WholeStageCodegen (241) - HashAggregate [channel,i_brand_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (323) - HashAggregate [channel,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel] #20 - WholeStageCodegen (322) - HashAggregate [channel,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (404) - HashAggregate [sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange #21 - WholeStageCodegen (403) - HashAggregate [sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #14 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #10 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #15 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #3 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #4 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #6 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #13 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id] #18 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] + CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id] #19 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] + CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel] #20 + CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] + CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange #21 + CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/explain.txt index e17c1e724e..30d0171b88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/explain.txt @@ -1,891 +1,805 @@ == Physical Plan == -TakeOrderedAndProject (157) -+- Union (156) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (23) - : : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : : :- * Project (17) - : : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.customer (11) - : : : : +- BroadcastExchange (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (18) - : : : +- BroadcastExchange (27) - : : : +- * Filter (26) - : : : +- * ColumnarToRow (25) - : : : +- Scan parquet spark_catalog.default.customer_address (24) - : : +- BroadcastExchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.date_dim (30) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * ColumnarToRow (38) - : +- Scan parquet spark_catalog.default.item (37) - :- * HashAggregate (72) - : +- Exchange (71) - : +- * HashAggregate (70) - : +- * Project (69) - : +- * BroadcastHashJoin Inner BuildRight (68) - : :- * Project (66) - : : +- * BroadcastHashJoin Inner BuildRight (65) - : : :- * Project (63) - : : : +- * BroadcastHashJoin Inner BuildRight (62) - : : : :- * Project (57) - : : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : : :- * Project (54) - : : : : : +- * BroadcastHashJoin Inner BuildRight (53) - : : : : : :- * Project (51) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : : : : :- * Filter (48) - : : : : : : : +- * ColumnarToRow (47) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (46) - : : : : : : +- ReusedExchange (49) - : : : : : +- ReusedExchange (52) - : : : : +- ReusedExchange (55) - : : : +- BroadcastExchange (61) - : : : +- * Filter (60) - : : : +- * ColumnarToRow (59) - : : : +- Scan parquet spark_catalog.default.customer_address (58) - : : +- ReusedExchange (64) - : +- ReusedExchange (67) - :- * HashAggregate (100) - : +- Exchange (99) - : +- * HashAggregate (98) - : +- * Project (97) - : +- * BroadcastHashJoin Inner BuildRight (96) - : :- * Project (94) - : : +- * BroadcastHashJoin Inner BuildRight (93) - : : :- * Project (91) - : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : :- * Project (84) - : : : : +- * BroadcastHashJoin Inner BuildRight (83) - : : : : :- * Project (81) - : : : : : +- * BroadcastHashJoin Inner BuildRight (80) - : : : : : :- * Project (78) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (77) - : : : : : : :- * Filter (75) - : : : : : : : +- * ColumnarToRow (74) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (73) - : : : : : : +- ReusedExchange (76) - : : : : : +- ReusedExchange (79) - : : : : +- ReusedExchange (82) - : : : +- BroadcastExchange (89) - : : : +- * Project (88) - : : : +- * Filter (87) - : : : +- * ColumnarToRow (86) - : : : +- Scan parquet spark_catalog.default.customer_address (85) - : : +- ReusedExchange (92) - : +- ReusedExchange (95) - :- * HashAggregate (128) - : +- Exchange (127) - : +- * HashAggregate (126) - : +- * Project (125) - : +- * BroadcastHashJoin Inner BuildRight (124) - : :- * Project (122) - : : +- * BroadcastHashJoin Inner BuildRight (121) - : : :- * Project (119) - : : : +- * BroadcastHashJoin Inner BuildRight (118) - : : : :- * Project (112) - : : : : +- * BroadcastHashJoin Inner BuildRight (111) - : : : : :- * Project (109) - : : : : : +- * BroadcastHashJoin Inner BuildRight (108) - : : : : : :- * Project (106) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (105) - : : : : : : :- * Filter (103) - : : : : : : : +- * ColumnarToRow (102) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (101) - : : : : : : +- ReusedExchange (104) - : : : : : +- ReusedExchange (107) - : : : : +- ReusedExchange (110) - : : : +- BroadcastExchange (117) - : : : +- * Project (116) - : : : +- * Filter (115) - : : : +- * ColumnarToRow (114) - : : : +- Scan parquet spark_catalog.default.customer_address (113) - : : +- ReusedExchange (120) - : +- ReusedExchange (123) - +- * HashAggregate (155) - +- Exchange (154) - +- * HashAggregate (153) - +- * Project (152) - +- * BroadcastHashJoin Inner BuildRight (151) - :- * Project (146) - : +- * BroadcastHashJoin Inner BuildRight (145) - : :- * Project (143) - : : +- * BroadcastHashJoin Inner BuildRight (142) - : : :- * Project (140) - : : : +- * BroadcastHashJoin Inner BuildRight (139) - : : : :- * Project (137) - : : : : +- * BroadcastHashJoin Inner BuildRight (136) - : : : : :- * Project (134) - : : : : : +- * BroadcastHashJoin Inner BuildRight (133) - : : : : : :- * Filter (131) - : : : : : : +- * ColumnarToRow (130) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (129) - : : : : : +- ReusedExchange (132) - : : : : +- ReusedExchange (135) - : : : +- ReusedExchange (138) - : : +- ReusedExchange (141) - : +- ReusedExchange (144) - +- BroadcastExchange (150) - +- * Filter (149) - +- * ColumnarToRow (148) - +- Scan parquet spark_catalog.default.item (147) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (146) ++- CometTakeOrderedAndProject (145) + +- CometUnion (144) + :- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometFilter (16) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometNativeScan parquet spark_catalog.default.item (32) + :- CometHashAggregate (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (51) + : : : : +- CometBroadcastHashJoin (50) + : : : : :- CometProject (48) + : : : : : +- CometBroadcastHashJoin (47) + : : : : : :- CometProject (45) + : : : : : : +- CometBroadcastHashJoin (44) + : : : : : : :- CometFilter (42) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (41) + : : : : : : +- ReusedExchange (43) + : : : : : +- ReusedExchange (46) + : : : : +- ReusedExchange (49) + : : : +- CometBroadcastExchange (55) + : : : +- CometProject (54) + : : : +- CometFilter (53) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (52) + : : +- ReusedExchange (58) + : +- ReusedExchange (61) + :- CometHashAggregate (92) + : +- CometExchange (91) + : +- CometHashAggregate (90) + : +- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometProject (86) + : : +- CometBroadcastHashJoin (85) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (77) + : : : : +- CometBroadcastHashJoin (76) + : : : : :- CometProject (74) + : : : : : +- CometBroadcastHashJoin (73) + : : : : : :- CometProject (71) + : : : : : : +- CometBroadcastHashJoin (70) + : : : : : : :- CometFilter (68) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (67) + : : : : : : +- ReusedExchange (69) + : : : : : +- ReusedExchange (72) + : : : : +- ReusedExchange (75) + : : : +- CometBroadcastExchange (81) + : : : +- CometProject (80) + : : : +- CometFilter (79) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (78) + : : +- ReusedExchange (84) + : +- ReusedExchange (87) + :- CometHashAggregate (118) + : +- CometExchange (117) + : +- CometHashAggregate (116) + : +- CometProject (115) + : +- CometBroadcastHashJoin (114) + : :- CometProject (112) + : : +- CometBroadcastHashJoin (111) + : : :- CometProject (109) + : : : +- CometBroadcastHashJoin (108) + : : : :- CometProject (103) + : : : : +- CometBroadcastHashJoin (102) + : : : : :- CometProject (100) + : : : : : +- CometBroadcastHashJoin (99) + : : : : : :- CometProject (97) + : : : : : : +- CometBroadcastHashJoin (96) + : : : : : : :- CometFilter (94) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (93) + : : : : : : +- ReusedExchange (95) + : : : : : +- ReusedExchange (98) + : : : : +- ReusedExchange (101) + : : : +- CometBroadcastExchange (107) + : : : +- CometProject (106) + : : : +- CometFilter (105) + : : : +- CometNativeScan parquet spark_catalog.default.customer_address (104) + : : +- ReusedExchange (110) + : +- ReusedExchange (113) + +- CometHashAggregate (143) + +- CometExchange (142) + +- CometHashAggregate (141) + +- CometProject (140) + +- CometBroadcastHashJoin (139) + :- CometProject (135) + : +- CometBroadcastHashJoin (134) + : :- CometProject (132) + : : +- CometBroadcastHashJoin (131) + : : :- CometProject (129) + : : : +- CometBroadcastHashJoin (128) + : : : :- CometProject (126) + : : : : +- CometBroadcastHashJoin (125) + : : : : :- CometProject (123) + : : : : : +- CometBroadcastHashJoin (122) + : : : : : :- CometFilter (120) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (119) + : : : : : +- ReusedExchange (121) + : : : : +- ReusedExchange (124) + : : : +- ReusedExchange (127) + : : +- ReusedExchange (130) + : +- ReusedExchange (133) + +- CometBroadcastExchange (138) + +- CometFilter (137) + +- CometNativeScan parquet spark_catalog.default.item (136) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(3) Filter [codegen id : 7] +(2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) -(6) Filter [codegen id : 1] +(5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = M)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#10)) +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#10, cd_dep_count#13] -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight -(10) Project [codegen id : 7] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +(8) CometProject Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(11) Scan parquet spark_catalog.default.customer +(9) CometNativeScan parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(11) CometProject Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight -(17) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(14) CometProject Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(18) Scan parquet spark_catalog.default.customer_demographics +(15) CometNativeScan parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [cd_demo_sk#19] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [cd_demo_sk#19] Condition : isnotnull(cd_demo_sk#19) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [cd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [cd_demo_sk#19] -(22) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight -(23) Project [codegen id : 7] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +(19) CometProject Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(24) Scan parquet spark_catalog.default.customer_address +(20) CometNativeScan parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) -(26) Filter [codegen id : 4] +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] -(27) BroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(29) Project [codegen id : 7] -Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(30) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +(26) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) -(32) Filter [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#24)) +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#24] -Input [2]: [d_date_sk#24, d_year#25] +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(34) BroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(36) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] - -(37) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] +(32) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] - -(39) Filter [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(40) BroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] - -(43) HashAggregate [codegen id : 7] -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Results [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] - -(44) Exchange -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 8] -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#63, avg(agg2#29)#64, avg(agg3#30)#65, avg(agg4#31)#66, avg(agg5#32)#67, avg(agg6#33)#68, avg(agg7#34)#69] -Results [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, avg(agg1#28)#63 AS agg1#70, avg(agg2#29)#64 AS agg2#71, avg(agg3#30)#65 AS agg3#72, avg(agg4#31)#66 AS agg4#73, avg(agg5#32)#67 AS agg5#74, avg(agg6#33)#68 AS agg6#75, avg(agg7#34)#69 AS agg7#76] - -(46) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] + +(38) CometHashAggregate +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(39) CometExchange +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(41) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 15] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(48) Filter [codegen id : 15] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(42) CometFilter +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) -(49) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] -(50) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight -(51) Project [codegen id : 15] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +(45) CometProject +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] -(52) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(46) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] -(53) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(47) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight -(54) Project [codegen id : 15] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(48) CometProject +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] -(55) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(49) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#70] -(56) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(50) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight -(57) Project [codegen id : 15] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +(51) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] -(58) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#20, ca_state#22, ca_country#23] +(52) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 12] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(60) Filter [codegen id : 12] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(61) BroadcastExchange -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(62) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 15] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_state#22, ca_country#23] - -(64) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(65) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 15] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23, d_date_sk#24] - -(67) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(68) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 15] -Output [10]: [i_item_id#27, ca_country#23, ca_state#22, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] - -(70) HashAggregate [codegen id : 15] -Input [10]: [i_item_id#27, ca_country#23, ca_state#22, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [3]: [i_item_id#27, ca_country#23, ca_state#22] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90] -Results [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] - -(71) Exchange -Input [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(72) HashAggregate [codegen id : 16] -Input [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] -Keys [3]: [i_item_id#27, ca_country#23, ca_state#22] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#105, avg(agg2#29)#106, avg(agg3#30)#107, avg(agg4#31)#108, avg(agg5#32)#109, avg(agg6#33)#110, avg(agg7#34)#111] -Results [11]: [i_item_id#27, ca_country#23, ca_state#22, null AS county#112, avg(agg1#28)#105 AS agg1#113, avg(agg2#29)#106 AS agg2#114, avg(agg3#30)#107 AS agg3#115, avg(agg4#31)#108 AS agg4#116, avg(agg5#32)#109 AS agg5#117, avg(agg6#33)#110 AS agg6#118, avg(agg7#34)#111 AS agg7#119] - -(73) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(53) CometFilter +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) + +(54) CometProject +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73], [ca_address_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) AS ca_state#24, ca_country#73] + +(55) CometBroadcastExchange +Input [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73] + +(56) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight + +(57) CometProject +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] + +(58) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#74] + +(59) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [1]: [d_date_sk#74] +Arguments: [cs_sold_date_sk#63], [d_date_sk#74], Inner, BuildRight + +(60) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, d_date_sk#74] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] + +(61) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#75, i_item_id#29] + +(62) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [2]: [i_item_sk#75, i_item_id#29] +Arguments: [cs_item_sk#57], [i_item_sk#75], Inner, BuildRight + +(63) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, i_item_sk#75, i_item_id#29] +Arguments: [i_item_id#29, ca_country#73, ca_state#24, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, ca_country#73, ca_state#24, cast(cs_quantity#58 as decimal(12,2)) AS agg1#34, cast(cs_list_price#59 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#38, cast(c_birth_year#69 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#40] + +(64) CometHashAggregate +Input [10]: [i_item_id#29, ca_country#73, ca_state#24, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(65) CometExchange +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Arguments: hashpartitioning(i_item_id#29, ca_country#73, ca_state#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(66) CometHashAggregate +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(67) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#98)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(74) ColumnarToRow [codegen id : 23] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(68) CometFilter +Input [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] +Condition : ((isnotnull(cs_bill_cdemo_sk#91) AND isnotnull(cs_bill_customer_sk#90)) AND isnotnull(cs_item_sk#92)) -(75) Filter [codegen id : 23] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(69) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#99, cd_dep_count#100] -(76) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(70) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] +Right output [2]: [cd_demo_sk#99, cd_dep_count#100] +Arguments: [cs_bill_cdemo_sk#91], [cd_demo_sk#99], Inner, BuildRight -(77) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(71) CometProject +Input [11]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_demo_sk#99, cd_dep_count#100] +Arguments: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100], [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100] -(78) Project [codegen id : 23] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +(72) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] -(79) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(73) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100] +Right output [4]: [c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Arguments: [cs_bill_customer_sk#90], [c_customer_sk#101], Inner, BuildRight -(80) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(74) CometProject +Input [13]: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] -(81) Project [codegen id : 23] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(75) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#105] -(82) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(76) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Right output [1]: [cd_demo_sk#105] +Arguments: [c_current_cdemo_sk#102], [cd_demo_sk#105], Inner, BuildRight -(83) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(77) CometProject +Input [12]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104, cd_demo_sk#105] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104] -(84) Project [codegen id : 23] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(85) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#20, ca_state#22, ca_country#23] +(78) CometNativeScan parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#106, ca_state#107, ca_country#108] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(86) ColumnarToRow [codegen id : 20] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(87) Filter [codegen id : 20] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(88) Project [codegen id : 20] -Output [2]: [ca_address_sk#20, ca_country#23] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(89) BroadcastExchange -Input [2]: [ca_address_sk#20, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(90) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 23] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_country#23] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_country#23] - -(92) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(93) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(94) Project [codegen id : 23] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_country#23] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_country#23, d_date_sk#24] - -(95) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(96) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(97) Project [codegen id : 23] -Output [9]: [i_item_id#27, ca_country#23, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_country#23, i_item_sk#26, i_item_id#27] - -(98) HashAggregate [codegen id : 23] -Input [9]: [i_item_id#27, ca_country#23, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [2]: [i_item_id#27, ca_country#23] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#120, count#121, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133] -Results [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] - -(99) Exchange -Input [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(100) HashAggregate [codegen id : 24] -Input [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] -Keys [2]: [i_item_id#27, ca_country#23] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#148, avg(agg2#29)#149, avg(agg3#30)#150, avg(agg4#31)#151, avg(agg5#32)#152, avg(agg6#33)#153, avg(agg7#34)#154] -Results [11]: [i_item_id#27, ca_country#23, null AS ca_state#155, null AS county#156, avg(agg1#28)#148 AS agg1#157, avg(agg2#29)#149 AS agg2#158, avg(agg3#30)#150 AS agg3#159, avg(agg4#31)#151 AS agg4#160, avg(agg5#32)#152 AS agg5#161, avg(agg6#33)#153 AS agg6#162, avg(agg7#34)#154 AS agg7#163] - -(101) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(79) CometFilter +Input [3]: [ca_address_sk#106, ca_state#107, ca_country#108] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#107, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#106)) + +(80) CometProject +Input [3]: [ca_address_sk#106, ca_state#107, ca_country#108] +Arguments: [ca_address_sk#106, ca_country#108], [ca_address_sk#106, ca_country#108] + +(81) CometBroadcastExchange +Input [2]: [ca_address_sk#106, ca_country#108] +Arguments: [ca_address_sk#106, ca_country#108] + +(82) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104] +Right output [2]: [ca_address_sk#106, ca_country#108] +Arguments: [c_current_addr_sk#103], [ca_address_sk#106], Inner, BuildRight + +(83) CometProject +Input [12]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104, ca_address_sk#106, ca_country#108] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108] + +(84) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#109] + +(85) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108] +Right output [1]: [d_date_sk#109] +Arguments: [cs_sold_date_sk#98], [d_date_sk#109], Inner, BuildRight + +(86) CometProject +Input [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108, d_date_sk#109] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108] + +(87) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#110, i_item_id#29] + +(88) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108] +Right output [2]: [i_item_sk#110, i_item_id#29] +Arguments: [cs_item_sk#92], [i_item_sk#110], Inner, BuildRight + +(89) CometProject +Input [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108, i_item_sk#110, i_item_id#29] +Arguments: [i_item_id#29, ca_country#108, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, ca_country#108, cast(cs_quantity#93 as decimal(12,2)) AS agg1#34, cast(cs_list_price#94 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#96 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#95 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#97 as decimal(12,2)) AS agg5#38, cast(c_birth_year#104 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#100 as decimal(12,2)) AS agg7#40] + +(90) CometHashAggregate +Input [9]: [i_item_id#29, ca_country#108, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [2]: [i_item_id#29, ca_country#108] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(91) CometExchange +Input [16]: [i_item_id#29, ca_country#108, sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124] +Arguments: hashpartitioning(i_item_id#29, ca_country#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(92) CometHashAggregate +Input [16]: [i_item_id#29, ca_country#108, sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124] +Keys [2]: [i_item_id#29, ca_country#108] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(93) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#133)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(102) ColumnarToRow [codegen id : 31] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(94) CometFilter +Input [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] +Condition : ((isnotnull(cs_bill_cdemo_sk#126) AND isnotnull(cs_bill_customer_sk#125)) AND isnotnull(cs_item_sk#127)) -(103) Filter [codegen id : 31] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(95) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#134, cd_dep_count#135] -(104) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(96) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] +Right output [2]: [cd_demo_sk#134, cd_dep_count#135] +Arguments: [cs_bill_cdemo_sk#126], [cd_demo_sk#134], Inner, BuildRight -(105) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(97) CometProject +Input [11]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_demo_sk#134, cd_dep_count#135] +Arguments: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135], [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135] -(106) Project [codegen id : 31] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(107) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(98) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] -(108) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(99) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135] +Right output [4]: [c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Arguments: [cs_bill_customer_sk#125], [c_customer_sk#136], Inner, BuildRight -(109) Project [codegen id : 31] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(100) CometProject +Input [13]: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] -(110) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(101) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#140] -(111) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(102) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Right output [1]: [cd_demo_sk#140] +Arguments: [c_current_cdemo_sk#137], [cd_demo_sk#140], Inner, BuildRight -(112) Project [codegen id : 31] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +(103) CometProject +Input [12]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139, cd_demo_sk#140] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139] -(113) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#22] +(104) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#141, ca_state#142] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(114) ColumnarToRow [codegen id : 28] -Input [2]: [ca_address_sk#20, ca_state#22] - -(115) Filter [codegen id : 28] -Input [2]: [ca_address_sk#20, ca_state#22] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(116) Project [codegen id : 28] -Output [1]: [ca_address_sk#20] -Input [2]: [ca_address_sk#20, ca_state#22] - -(117) BroadcastExchange -Input [1]: [ca_address_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(118) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(119) Project [codegen id : 31] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20] - -(120) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(121) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(122) Project [codegen id : 31] -Output [8]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, d_date_sk#24] - -(123) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(124) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(125) Project [codegen id : 31] -Output [8]: [i_item_id#27, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_sk#26, i_item_id#27] - -(126) HashAggregate [codegen id : 31] -Input [8]: [i_item_id#27, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [1]: [i_item_id#27] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#164, count#165, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177] -Results [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] - -(127) Exchange -Input [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] -Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(128) HashAggregate [codegen id : 32] -Input [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] -Keys [1]: [i_item_id#27] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#192, avg(agg2#29)#193, avg(agg3#30)#194, avg(agg4#31)#195, avg(agg5#32)#196, avg(agg6#33)#197, avg(agg7#34)#198] -Results [11]: [i_item_id#27, null AS ca_country#199, null AS ca_state#200, null AS county#201, avg(agg1#28)#192 AS agg1#202, avg(agg2#29)#193 AS agg2#203, avg(agg3#30)#194 AS agg3#204, avg(agg4#31)#195 AS agg4#205, avg(agg5#32)#196 AS agg5#206, avg(agg6#33)#197 AS agg6#207, avg(agg7#34)#198 AS agg7#208] - -(129) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(105) CometFilter +Input [2]: [ca_address_sk#141, ca_state#142] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#142, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#141)) + +(106) CometProject +Input [2]: [ca_address_sk#141, ca_state#142] +Arguments: [ca_address_sk#141], [ca_address_sk#141] + +(107) CometBroadcastExchange +Input [1]: [ca_address_sk#141] +Arguments: [ca_address_sk#141] + +(108) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139] +Right output [1]: [ca_address_sk#141] +Arguments: [c_current_addr_sk#138], [ca_address_sk#141], Inner, BuildRight + +(109) CometProject +Input [11]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139, ca_address_sk#141] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139] + +(110) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#143] + +(111) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139] +Right output [1]: [d_date_sk#143] +Arguments: [cs_sold_date_sk#133], [d_date_sk#143], Inner, BuildRight + +(112) CometProject +Input [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139, d_date_sk#143] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139] + +(113) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#144, i_item_id#29] + +(114) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139] +Right output [2]: [i_item_sk#144, i_item_id#29] +Arguments: [cs_item_sk#127], [i_item_sk#144], Inner, BuildRight + +(115) CometProject +Input [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139, i_item_sk#144, i_item_id#29] +Arguments: [i_item_id#29, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, cast(cs_quantity#128 as decimal(12,2)) AS agg1#34, cast(cs_list_price#129 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#131 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#130 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#132 as decimal(12,2)) AS agg5#38, cast(c_birth_year#139 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#135 as decimal(12,2)) AS agg7#40] + +(116) CometHashAggregate +Input [8]: [i_item_id#29, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [1]: [i_item_id#29] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(117) CometExchange +Input [15]: [i_item_id#29, sum#145, count#146, sum#147, count#148, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(118) CometHashAggregate +Input [15]: [i_item_id#29, sum#145, count#146, sum#147, count#148, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158] +Keys [1]: [i_item_id#29] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(119) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#167)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(130) ColumnarToRow [codegen id : 39] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(131) Filter [codegen id : 39] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(120) CometFilter +Input [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] +Condition : ((isnotnull(cs_bill_cdemo_sk#160) AND isnotnull(cs_bill_customer_sk#159)) AND isnotnull(cs_item_sk#161)) -(132) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(121) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#168, cd_dep_count#169] -(133) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(122) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] +Right output [2]: [cd_demo_sk#168, cd_dep_count#169] +Arguments: [cs_bill_cdemo_sk#160], [cd_demo_sk#168], Inner, BuildRight -(134) Project [codegen id : 39] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +(123) CometProject +Input [11]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_demo_sk#168, cd_dep_count#169] +Arguments: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169], [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169] -(135) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(124) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] -(136) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(125) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169] +Right output [4]: [c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Arguments: [cs_bill_customer_sk#159], [c_customer_sk#170], Inner, BuildRight -(137) Project [codegen id : 39] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(126) CometProject +Input [13]: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] -(138) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(127) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#174] -(139) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(128) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Right output [1]: [cd_demo_sk#174] +Arguments: [c_current_cdemo_sk#171], [cd_demo_sk#174], Inner, BuildRight -(140) Project [codegen id : 39] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +(129) CometProject +Input [12]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173, cd_demo_sk#174] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173] -(141) ReusedExchange [Reuses operator id: 117] -Output [1]: [ca_address_sk#20] +(130) ReusedExchange [Reuses operator id: 107] +Output [1]: [ca_address_sk#175] -(142) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None +(131) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173] +Right output [1]: [ca_address_sk#175] +Arguments: [c_current_addr_sk#172], [ca_address_sk#175], Inner, BuildRight -(143) Project [codegen id : 39] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20] +(132) CometProject +Input [11]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173, ca_address_sk#175] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173] -(144) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] +(133) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#176] -(145) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None +(134) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173] +Right output [1]: [d_date_sk#176] +Arguments: [cs_sold_date_sk#167], [d_date_sk#176], Inner, BuildRight -(146) Project [codegen id : 39] -Output [8]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, d_date_sk#24] +(135) CometProject +Input [10]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173, d_date_sk#176] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173] -(147) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#26] +(136) CometNativeScan parquet spark_catalog.default.item +Output [1]: [i_item_sk#177] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(148) ColumnarToRow [codegen id : 38] -Input [1]: [i_item_sk#26] - -(149) Filter [codegen id : 38] -Input [1]: [i_item_sk#26] -Condition : isnotnull(i_item_sk#26) +(137) CometFilter +Input [1]: [i_item_sk#177] +Condition : isnotnull(i_item_sk#177) -(150) BroadcastExchange -Input [1]: [i_item_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +(138) CometBroadcastExchange +Input [1]: [i_item_sk#177] +Arguments: [i_item_sk#177] -(151) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(139) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173] +Right output [1]: [i_item_sk#177] +Arguments: [cs_item_sk#161], [i_item_sk#177], Inner, BuildRight -(152) Project [codegen id : 39] -Output [7]: [cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_sk#26] +(140) CometProject +Input [9]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173, i_item_sk#177] +Arguments: [agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [cast(cs_quantity#162 as decimal(12,2)) AS agg1#34, cast(cs_list_price#163 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#165 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#164 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#166 as decimal(12,2)) AS agg5#38, cast(c_birth_year#173 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#169 as decimal(12,2)) AS agg7#40] -(153) HashAggregate [codegen id : 39] -Input [7]: [agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] +(141) CometHashAggregate +Input [7]: [agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] Keys: [] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216, sum#217, count#218, sum#219, count#220, sum#221, count#222] -Results [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] -(154) Exchange -Input [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(142) CometExchange +Input [14]: [sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(155) HashAggregate [codegen id : 40] -Input [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] +(143) CometHashAggregate +Input [14]: [sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] Keys: [] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#237, avg(agg2#29)#238, avg(agg3#30)#239, avg(agg4#31)#240, avg(agg5#32)#241, avg(agg6#33)#242, avg(agg7#34)#243] -Results [11]: [null AS i_item_id#244, null AS ca_country#245, null AS ca_state#246, null AS county#247, avg(agg1#28)#237 AS agg1#248, avg(agg2#29)#238 AS agg2#249, avg(agg3#30)#239 AS agg3#250, avg(agg4#31)#240 AS agg4#251, avg(agg5#32)#241 AS agg5#252, avg(agg6#33)#242 AS agg6#253, avg(agg7#34)#243 AS agg7#254] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(144) CometUnion +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] +Child 1 Input [11]: [i_item_id#29, ca_country#73, ca_state#24, county#199, agg1#200, agg2#201, agg3#202, agg4#203, agg5#204, agg6#205, agg7#206] +Child 2 Input [11]: [i_item_id#29, ca_country#108, ca_state#207, county#208, agg1#209, agg2#210, agg3#211, agg4#212, agg5#213, agg6#214, agg7#215] +Child 3 Input [11]: [i_item_id#29, ca_country#216, ca_state#217, county#218, agg1#219, agg2#220, agg3#221, agg4#222, agg5#223, agg6#224, agg7#225] +Child 4 Input [11]: [i_item_id#226, ca_country#227, ca_state#228, county#229, agg1#230, agg2#231, agg3#232, agg4#233, agg5#234, agg6#235, agg7#236] -(156) Union +(145) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#192,agg2#193,agg3#194,agg4#195,agg5#196,agg6#197,agg7#198]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] -(157) TakeOrderedAndProject -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#70, agg2#71, agg3#72, agg4#73, agg5#74, agg6#75, agg7#76] -Arguments: 100, [ca_country#23 ASC NULLS FIRST, ca_state#22 ASC NULLS FIRST, ca_county#21 ASC NULLS FIRST, i_item_id#27 ASC NULLS FIRST], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#70, agg2#71, agg3#72, agg4#73, agg5#74, agg6#75, agg7#76] +(146) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/extended.txt new file mode 100644 index 0000000000..59e46c3c8e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/extended.txt @@ -0,0 +1,204 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer_address + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 201 out of 201 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/simplified.txt index fa297f1803..a5655241a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/simplified.txt @@ -1,227 +1,148 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - Union - WholeStageCodegen (8) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - WholeStageCodegen (16) - HashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state] #8 - WholeStageCodegen (15) - HashAggregate [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,ca_state,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (12) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (24) - HashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country] #10 - WholeStageCodegen (23) - HashAggregate [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (20) - Project [ca_address_sk,ca_country] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (32) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #12 - WholeStageCodegen (31) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (28) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (40) - HashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange #14 - WholeStageCodegen (39) - HashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - ReusedExchange [ca_address_sk] #13 - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (38) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state] #8 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 + CometProject [ca_state] [ca_address_sk,ca_state,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country] #10 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_country] #11 + CometProject [ca_address_sk,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id] #12 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk] #13 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange #14 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + ReusedExchange [ca_address_sk] #13 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [i_item_sk] #15 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/explain.txt index 8d513788d8..23634911fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/explain.txt @@ -1,140 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.catalog_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16] -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/extended.txt new file mode 100644 index 0000000000..7fd40cf7dd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/extended.txt @@ -0,0 +1,25 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 19 out of 22 eligible operators (86%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/simplified.txt index 1e7108fcba..06e56fab81 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/explain.txt index 174908acf7..6c7ac14107 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/explain.txt @@ -1,151 +1,150 @@ == Physical Plan == TakeOrderedAndProject (26) +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Expand (22) - +- * Project (21) - +- * BroadcastNestedLoopJoin Inner BuildRight (20) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.warehouse (17) - - -(1) Scan parquet spark_catalog.default.inventory + +- * CometColumnarToRow (24) + +- CometColumnarExchange (23) + +- * HashAggregate (22) + +- * Expand (21) + +- * Project (20) + +- * BroadcastNestedLoopJoin Inner BuildRight (19) + :- * CometColumnarToRow (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + +- BroadcastExchange (18) + +- * CometColumnarToRow (17) + +- CometNativeScan parquet spark_catalog.default.warehouse (16) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#3)] PushedFilters: [IsNotNull(inv_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 4] -Output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] +(8) CometProject Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Condition : isnotnull(i_item_sk#6) -(14) BroadcastExchange +(11) CometProject Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#7, 50, true, false, true) AS i_brand#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#10, 50, true, false, true) AS i_product_name#14] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(16) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(13) CometBroadcastHashJoin +Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight -(17) Scan parquet spark_catalog.default.warehouse +(14) CometProject +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(15) CometColumnarToRow [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(16) CometNativeScan parquet spark_catalog.default.warehouse Output: [] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] ReadSchema: struct<> -(18) ColumnarToRow [codegen id : 3] +(17) CometColumnarToRow [codegen id : 1] Input: [] -(19) BroadcastExchange +(18) BroadcastExchange Input: [] -Arguments: IdentityBroadcastMode, [plan_id=3] +Arguments: IdentityBroadcastMode, [plan_id=1] -(20) BroadcastNestedLoopJoin [codegen id : 4] +(19) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(21) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(20) Project [codegen id : 2] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(22) Expand [codegen id : 4] -Input [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Arguments: [[inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9, 0], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, null, 1], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, null, null, 3], [inv_quantity_on_hand#2, i_product_name#10, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(21) Expand [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] -(23) HashAggregate [codegen id : 4] -Input [6]: [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(22) HashAggregate [codegen id : 2] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#16, count#17] -Results [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(23) CometColumnarExchange +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(24) Exchange -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(24) CometColumnarToRow [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] -(25) HashAggregate [codegen id : 5] -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(25) HashAggregate [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#20] -Results [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, avg(inv_quantity_on_hand#2)#20 AS qoh#21] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] (26) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] -Arguments: 100, [qoh#21 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#12 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_category#14 ASC NULLS FIRST], [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/extended.txt new file mode 100644 index 0000000000..e4737a8934 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/extended.txt @@ -0,0 +1,28 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Expand + +- Project + +- BroadcastNestedLoopJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- BroadcastExchange + +- CometColumnarToRow + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 16 out of 23 eligible operators (69%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/simplified.txt index 526ebfb48e..199ad41e45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/simplified.txt @@ -1,39 +1,33 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (5) + WholeStageCodegen (3) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + CometColumnarToRow InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometNativeScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/explain.txt index 33ed9945bb..e33cea1827 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/explain.txt @@ -1,305 +1,268 @@ == Physical Plan == -TakeOrderedAndProject (49) -+- Union (48) - :- * HashAggregate (27) - : +- * HashAggregate (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.item (11) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.warehouse (17) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * HashAggregate (29) - : +- ReusedExchange (28) - :- * HashAggregate (37) - : +- Exchange (36) - : +- * HashAggregate (35) - : +- * HashAggregate (34) - : +- ReusedExchange (33) - :- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * HashAggregate (39) - : +- ReusedExchange (38) - +- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * HashAggregate (44) - +- ReusedExchange (43) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometUnion (45) + :- CometHashAggregate (24) + : +- CometHashAggregate (23) + : +- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.warehouse (15) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometHashAggregate (26) + : +- ReusedExchange (25) + :- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + :- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometHashAggregate (36) + : +- ReusedExchange (35) + +- CometHashAggregate (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometHashAggregate (41) + +- ReusedExchange (40) + + +(1) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 4] -Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +(8) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] +Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Condition : isnotnull(i_item_sk#7) -(14) BroadcastExchange +(11) CometProject Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(16) Project [codegen id : 4] -Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(17) Scan parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#12] +(15) CometNativeScan parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [1]: [w_warehouse_sk#12] +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) -(19) Filter [codegen id : 3] -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] -(20) BroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#12] -Join type: Inner -Join condition: None +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] - -(23) HashAggregate [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] +(20) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#13, count#14] -Results [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -(24) Exchange -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, i_category#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 5] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(26) HashAggregate [codegen id : 5] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#18] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#19, count#20] -Results [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#21, count#22] - -(27) HashAggregate [codegen id : 5] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#21, count#22] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#23] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, avg(qoh#18)#23 AS qoh#24] - -(28) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(29) HashAggregate [codegen id : 10] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [4]: [i_product_name#11, i_brand#8, i_class#9, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(30) HashAggregate [codegen id : 10] -Input [4]: [i_product_name#11, i_brand#8, i_class#9, qoh#18] -Keys [3]: [i_product_name#11, i_brand#8, i_class#9] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#25, count#26] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] - -(31) Exchange -Input [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 11] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] -Keys [3]: [i_product_name#11, i_brand#8, i_class#9] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#29] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, null AS i_category#30, avg(qoh#18)#29 AS qoh#31] - -(33) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(34) HashAggregate [codegen id : 16] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] +(22) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [3]: [i_product_name#11, i_brand#8, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(35) HashAggregate [codegen id : 16] -Input [3]: [i_product_name#11, i_brand#8, qoh#18] -Keys [2]: [i_product_name#11, i_brand#8] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#32, count#33] -Results [4]: [i_product_name#11, i_brand#8, sum#34, count#35] - -(36) Exchange -Input [4]: [i_product_name#11, i_brand#8, sum#34, count#35] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(37) HashAggregate [codegen id : 17] -Input [4]: [i_product_name#11, i_brand#8, sum#34, count#35] -Keys [2]: [i_product_name#11, i_brand#8] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#36] -Results [5]: [i_product_name#11, i_brand#8, null AS i_class#37, null AS i_category#38, avg(qoh#18)#36 AS qoh#39] - -(38) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(39) HashAggregate [codegen id : 22] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [2]: [i_product_name#11, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(40) HashAggregate [codegen id : 22] -Input [2]: [i_product_name#11, qoh#18] -Keys [1]: [i_product_name#11] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#40, count#41] -Results [3]: [i_product_name#11, sum#42, count#43] - -(41) Exchange -Input [3]: [i_product_name#11, sum#42, count#43] -Arguments: hashpartitioning(i_product_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 23] -Input [3]: [i_product_name#11, sum#42, count#43] -Keys [1]: [i_product_name#11] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#44] -Results [5]: [i_product_name#11, null AS i_brand#45, null AS i_class#46, null AS i_category#47, avg(qoh#18)#44 AS qoh#48] - -(43) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(44) HashAggregate [codegen id : 28] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [1]: [avg(inv_quantity_on_hand#3)#17 AS qoh#18] -(45) HashAggregate [codegen id : 28] -Input [1]: [qoh#18] +(23) CometHashAggregate +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#23] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [partial_avg(qoh#23)] + +(24) CometHashAggregate +Input [6]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, sum#24, count#25] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [avg(qoh#23)] + +(25) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] + +(26) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#28)] + +(27) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, i_class#13, qoh#23] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [partial_avg(qoh#23)] + +(28) CometExchange +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#29, count#30] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometHashAggregate +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#29, count#30] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [avg(qoh#23)] + +(30) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#31, count#32] + +(31) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#31, count#32] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#33)] + +(32) CometHashAggregate +Input [3]: [i_product_name#15, i_brand#12, qoh#23] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [partial_avg(qoh#23)] + +(33) CometExchange +Input [4]: [i_product_name#15, i_brand#12, sum#34, count#35] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, sum#34, count#35] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [avg(qoh#23)] + +(35) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#36, count#37] + +(36) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#36, count#37] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#38)] + +(37) CometHashAggregate +Input [2]: [i_product_name#15, qoh#23] +Keys [1]: [i_product_name#15] +Functions [1]: [partial_avg(qoh#23)] + +(38) CometExchange +Input [3]: [i_product_name#15, sum#39, count#40] +Arguments: hashpartitioning(i_product_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometHashAggregate +Input [3]: [i_product_name#15, sum#39, count#40] +Keys [1]: [i_product_name#15] +Functions [1]: [avg(qoh#23)] + +(40) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#41, count#42] + +(41) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#41, count#42] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#43)] + +(42) CometHashAggregate +Input [1]: [qoh#23] Keys: [] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#49, count#50] -Results [2]: [sum#51, count#52] +Functions [1]: [partial_avg(qoh#23)] -(46) Exchange -Input [2]: [sum#51, count#52] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(43) CometExchange +Input [2]: [sum#44, count#45] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(47) HashAggregate [codegen id : 29] -Input [2]: [sum#51, count#52] +(44) CometHashAggregate +Input [2]: [sum#44, count#45] Keys: [] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#53] -Results [5]: [null AS i_product_name#54, null AS i_brand#55, null AS i_class#56, null AS i_category#57, avg(qoh#18)#53 AS qoh#58] +Functions [1]: [avg(qoh#23)] + +(45) CometUnion +Child 0 Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] +Child 1 Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#47, qoh#48] +Child 2 Input [5]: [i_product_name#15, i_brand#12, i_class#49, i_category#50, qoh#51] +Child 3 Input [5]: [i_product_name#15, i_brand#52, i_class#53, i_category#54, qoh#55] +Child 4 Input [5]: [i_product_name#56, i_brand#57, i_class#58, i_category#59, qoh#60] -(48) Union +(46) CometTakeOrderedAndProject +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#46 ASC NULLS FIRST,i_product_name#19 ASC NULLS FIRST,i_brand#20 ASC NULLS FIRST,i_class#21 ASC NULLS FIRST,i_category#22 ASC NULLS FIRST], output=[i_product_name#19,i_brand#20,i_class#21,i_category#22,qoh#46]), [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46], 100, 0, [qoh#46 ASC NULLS FIRST, i_product_name#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_class#21 ASC NULLS FIRST, i_category#22 ASC NULLS FIRST], [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] -(49) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#24] -Arguments: 100, [qoh#24 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#8 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#24] +(47) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/extended.txt new file mode 100644 index 0000000000..ed898ac4d5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/extended.txt @@ -0,0 +1,129 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.warehouse + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.warehouse + +Comet accelerated 126 out of 126 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/simplified.txt index 04c8e759d1..d404363103 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/simplified.txt @@ -1,78 +1,49 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - Union - WholeStageCodegen (5) - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(qoh),qoh,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,inv_quantity_on_hand] [sum,count,sum,count] - Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk] - WholeStageCodegen (11) - HashAggregate [i_product_name,i_brand,i_class,sum,count] [avg(qoh),i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class] #5 - WholeStageCodegen (10) - HashAggregate [i_product_name,i_brand,i_class,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (17) - HashAggregate [i_product_name,i_brand,sum,count] [avg(qoh),i_class,i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand] #6 - WholeStageCodegen (16) - HashAggregate [i_product_name,i_brand,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (23) - HashAggregate [i_product_name,sum,count] [avg(qoh),i_brand,i_class,i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name] #7 - WholeStageCodegen (22) - HashAggregate [i_product_name,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (29) - HashAggregate [sum,count] [avg(qoh),i_product_name,i_brand,i_class,i_category,qoh,sum,count] - InputAdapter - Exchange #8 - WholeStageCodegen (28) - HashAggregate [qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometUnion [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,i_category,sum,count] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,i_product_name,i_brand,i_class,i_category,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,sum,count] + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand,i_class] #5 + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] + CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand] #6 + CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] + CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name] #7 + CometHashAggregate [qoh] [i_product_name,sum,count] + CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange #8 + CometHashAggregate [qoh] [sum,count] + CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/explain.txt index a0e507ac1c..6af787c781 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/explain.txt @@ -1,437 +1,453 @@ == Physical Plan == -* Sort (48) -+- Exchange (47) - +- * Filter (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- * Filter (47) + +- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometNativeScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometNativeScan parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometNativeScan parquet spark_catalog.default.customer_address (32) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(5) Exchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.store_returns +(6) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(8) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(11) Exchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +(12) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(15) Scan parquet spark_catalog.default.store +(13) CometNativeScan parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] +(14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(28) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(25) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(26) CometFilter +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Condition : ((isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) AND isnotnull(c_birth_country#30)) -(30) Filter [codegen id : 7] -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) +(27) CometProject +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [c_customer_sk#26, c_current_addr_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#32, c_birth_country#30] -(31) BroadcastExchange -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(33) Project [codegen id : 9] -Output [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(30) CometProject +Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] -(34) Scan parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(31) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(32) CometNativeScan parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(33) CometFilter +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Condition : ((isnotnull(ca_address_sk#33) AND isnotnull(ca_country#36)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true))) + +(34) CometProject +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Arguments: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36], [ca_address_sk#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#34, 2, true, false, true) AS ca_state#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true) AS ca_zip#38, ca_country#36] -(36) Filter [codegen id : 8] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) +(35) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] -(37) BroadcastExchange -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=6] +(36) BroadcastExchange +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] +Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[3, string, true]), input[2, string, true]),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#27, c_birth_country#30, s_zip#16] +Right keys [3]: [ca_address_sk#33, upper(ca_country#36), ca_zip#38] Join type: Inner Join condition: None -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30, ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#30] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] +Aggregate Attributes [1]: [sum#39] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] + +(40) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(41) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [partial_sum(netpaid#33)] -Aggregate Attributes [2]: [sum#34, isEmpty#35] -Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(44) Exchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [sum(netpaid#33)] -Aggregate Attributes [1]: [sum(netpaid#33)#38] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) - -(47) Exchange -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(48) Sort [codegen id : 12] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST], true, 0 +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#41] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#41,17,2) AS netpaid#42] -===== Subqueries ===== +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, netpaid#42] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [partial_sum(netpaid#42)] +Aggregate Attributes [2]: [sum#43, isEmpty#44] +Results [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (75) -+- Exchange (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- Exchange (71) - +- * HashAggregate (70) - +- * Project (69) - +- * BroadcastHashJoin Inner BuildRight (68) - :- * Project (66) - : +- * BroadcastHashJoin Inner BuildRight (65) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * Project (57) - : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : :- * Project (54) - : : : : +- * SortMergeJoin Inner (53) - : : : : :- * Sort (50) - : : : : : +- ReusedExchange (49) - : : : : +- * Sort (52) - : : : : +- ReusedExchange (51) - : : : +- ReusedExchange (55) - : : +- BroadcastExchange (61) - : : +- * Filter (60) - : : +- * ColumnarToRow (59) - : : +- Scan parquet spark_catalog.default.item (58) - : +- ReusedExchange (64) - +- ReusedExchange (67) - - -(49) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(50) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +(44) CometColumnarExchange +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(51) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] -(52) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [sum(netpaid#42)] +Aggregate Attributes [1]: [sum(netpaid#42)#47] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, sum(netpaid#42)#47 AS paid#48] -(53) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Condition : (isnotnull(paid#48) AND (cast(paid#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8)))) -(54) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +(48) CometColumnarExchange +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: rangepartitioning(c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(55) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(49) CometSort +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48], [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] -(56) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(50) CometColumnarToRow [codegen id : 5] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] -(57) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +===== Subqueries ===== -(58) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* HashAggregate (80) ++- * CometColumnarToRow (79) + +- CometColumnarExchange (78) + +- * HashAggregate (77) + +- * HashAggregate (76) + +- * CometColumnarToRow (75) + +- CometColumnarExchange (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- * CometColumnarToRow (69) + : +- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (56) + : : : : +- CometSortMergeJoin (55) + : : : : :- CometSort (52) + : : : : : +- ReusedExchange (51) + : : : : +- CometSort (54) + : : : : +- ReusedExchange (53) + : : : +- ReusedExchange (57) + : : +- CometBroadcastExchange (63) + : : +- CometProject (62) + : : +- CometFilter (61) + : : +- CometNativeScan parquet spark_catalog.default.item (60) + : +- ReusedExchange (66) + +- ReusedExchange (70) + + +(51) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] + +(52) CometSort +Input [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55], [ss_ticket_number#54 ASC NULLS FIRST, ss_item_sk#51 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#56, sr_ticket_number#57] + +(54) CometSort +Input [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [sr_item_sk#56, sr_ticket_number#57], [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Right output [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_ticket_number#54, ss_item_sk#51], [sr_ticket_number#57, sr_item_sk#56], Inner + +(56) CometProject +Input [7]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55, sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55], [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] + +(57) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] + +(58) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] +Right output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_store_sk#53], [s_store_sk#58], Inner, BuildRight + +(59) CometProject +Input [8]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55, s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16], [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] + +(60) CometNativeScan parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(61) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) -(60) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) +(62) CometProject +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [i_item_sk#60, i_current_price#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#62, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#63, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#64, 10, true, false, true) AS i_units#25, i_manager_id#65] -(61) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(63) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(62) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(64) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_item_sk#51], [i_item_sk#60], Inner, BuildRight -(63) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(65) CometProject +Input [12]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(64) ReusedExchange [Reuses operator id: 31] -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(66) ReusedExchange [Reuses operator id: 28] +Output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(65) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(67) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Right output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_customer_sk#52], [c_customer_sk#66], Inner, BuildRight + +(68) CometProject +Input [15]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68], [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(66) Project [codegen id : 9] -Output [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(69) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(67) ReusedExchange [Reuses operator id: 37] -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(70) ReusedExchange [Reuses operator id: 36] +Output [4]: [ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] -(68) BroadcastHashJoin [codegen id : 9] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] +(71) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#67, c_birth_country#68, s_zip#16] +Right keys [3]: [ca_address_sk#69, upper(ca_country#70), ca_zip#38] Join type: Inner Join condition: None -(69) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(70) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#42] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] - -(71) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(72) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(73) HashAggregate [codegen id : 10] -Input [1]: [netpaid#33] +(72) Project [codegen id : 2] +Output [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68, ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] + +(73) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum#71] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(74) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(75) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(76) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#55))#41] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#55))#41,17,2) AS netpaid#42] + +(77) HashAggregate [codegen id : 3] +Input [1]: [netpaid#42] Keys: [] -Functions [1]: [partial_avg(netpaid#33)] -Aggregate Attributes [2]: [sum#44, count#45] -Results [2]: [sum#46, count#47] +Functions [1]: [partial_avg(netpaid#42)] +Aggregate Attributes [2]: [sum#73, count#74] +Results [2]: [sum#75, count#76] + +(78) CometColumnarExchange +Input [2]: [sum#75, count#76] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] -(74) Exchange -Input [2]: [sum#46, count#47] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(79) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#75, count#76] -(75) HashAggregate [codegen id : 11] -Input [2]: [sum#46, count#47] +(80) HashAggregate [codegen id : 4] +Input [2]: [sum#75, count#76] Keys: [] -Functions [1]: [avg(netpaid#33)] -Aggregate Attributes [1]: [avg(netpaid#33)#48] -Results [1]: [(0.05 * avg(netpaid#33)#48) AS (0.05 * avg(netpaid))#49] +Functions [1]: [avg(netpaid#42)] +Aggregate Attributes [1]: [avg(netpaid#42)#77] +Results [1]: [(0.05 * avg(netpaid#42)#77) AS (0.05 * avg(netpaid))#78] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/extended.txt new file mode 100644 index 0000000000..13317e59ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/extended.txt @@ -0,0 +1,99 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Filter + : +- Subquery + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_address + +Comet accelerated 72 out of 88 eligible operators (81%). Final plan contains 9 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/simplified.txt index 104ec43154..b24e51723a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/simplified.txt @@ -1,122 +1,99 @@ -WholeStageCodegen (12) - Sort [c_last_name,c_first_name,s_store_name] +WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (11) - Filter [paid] - Subquery #1 - WholeStageCodegen (11) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometSort [c_last_name,c_first_name,s_store_name,paid] + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (4) + Filter [paid] + Subquery #1 + WholeStageCodegen (4) + HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow InputAdapter - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #2 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #4 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #5 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk,c_current_addr_sk,c_birth_country] - ColumnarToRow + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/explain.txt index 25c5db89ce..db9e26b55f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/explain.txt @@ -1,443 +1,400 @@ == Physical Plan == -TakeOrderedAndProject (77) -+- Union (76) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (17) - : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : +- BroadcastExchange (15) - : : : +- * Project (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- BroadcastExchange (21) - : : +- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.store (18) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.item (24) - :- * HashAggregate (54) - : +- Exchange (53) - : +- * HashAggregate (52) - : +- * Project (51) - : +- * BroadcastHashJoin Inner BuildRight (50) - : :- * Project (48) - : : +- * BroadcastHashJoin Inner BuildRight (47) - : : :- * Project (41) - : : : +- * BroadcastHashJoin Inner BuildRight (40) - : : : :- * Project (38) - : : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : : :- * Filter (35) - : : : : : +- * ColumnarToRow (34) - : : : : : +- Scan parquet spark_catalog.default.store_sales (33) - : : : : +- ReusedExchange (36) - : : : +- ReusedExchange (39) - : : +- BroadcastExchange (46) - : : +- * Project (45) - : : +- * Filter (44) - : : +- * ColumnarToRow (43) - : : +- Scan parquet spark_catalog.default.store (42) - : +- ReusedExchange (49) - +- * HashAggregate (75) - +- Exchange (74) - +- * HashAggregate (73) - +- * Project (72) - +- * BroadcastHashJoin Inner BuildRight (71) - :- * Project (66) - : +- * BroadcastHashJoin Inner BuildRight (65) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * Project (60) - : : : +- * BroadcastHashJoin Inner BuildRight (59) - : : : :- * Filter (57) - : : : : +- * ColumnarToRow (56) - : : : : +- Scan parquet spark_catalog.default.store_sales (55) - : : : +- ReusedExchange (58) - : : +- ReusedExchange (61) - : +- ReusedExchange (64) - +- BroadcastExchange (70) - +- * Filter (69) - +- * ColumnarToRow (68) - +- Scan parquet spark_catalog.default.item (67) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometUnion (69) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.store (15) + : +- CometBroadcastExchange (24) + : +- CometProject (23) + : +- CometFilter (22) + : +- CometNativeScan parquet spark_catalog.default.item (21) + :- CometHashAggregate (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (37) + : : : +- CometBroadcastHashJoin (36) + : : : :- CometProject (34) + : : : : +- CometBroadcastHashJoin (33) + : : : : :- CometFilter (31) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (30) + : : : : +- ReusedExchange (32) + : : : +- ReusedExchange (35) + : : +- CometBroadcastExchange (41) + : : +- CometProject (40) + : : +- CometFilter (39) + : : +- CometNativeScan parquet spark_catalog.default.store (38) + : +- ReusedExchange (44) + +- CometHashAggregate (68) + +- CometExchange (67) + +- CometHashAggregate (66) + +- CometProject (65) + +- CometBroadcastHashJoin (64) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- ReusedExchange (58) + +- CometBroadcastExchange (63) + +- CometFilter (62) + +- CometNativeScan parquet spark_catalog.default.item (61) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometNativeScan parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = F)) AND (cd_marital_status#11 = W)) AND (cd_education_status#12 = Primary )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(8) CometProject Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +(14) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(18) Scan parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] +(16) CometFilter Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(20) Filter [codegen id : 3] +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] -(21) BroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(24) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] +(21) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] - -(26) Filter [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(27) BroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [6]: [i_item_id#18, s_state#16, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] - -(30) HashAggregate [codegen id : 5] -Input [6]: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Results [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] - -(31) Exchange -Input [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -Arguments: hashpartitioning(i_item_id#18, s_state#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#39, avg(UnscaledValue(agg2#20))#40, avg(UnscaledValue(agg3#21))#41, avg(UnscaledValue(agg4#22))#42] -Results [7]: [i_item_id#18, s_state#16, 0 AS g_state#43, avg(agg1#19)#39 AS agg1#44, cast((avg(UnscaledValue(agg2#20))#40 / 100.0) as decimal(11,6)) AS agg2#45, cast((avg(UnscaledValue(agg3#21))#41 / 100.0) as decimal(11,6)) AS agg3#46, cast((avg(UnscaledValue(agg4#22))#42 / 100.0) as decimal(11,6)) AS agg4#47] - -(33) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) + +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] + +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] + +(27) CometHashAggregate +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(28) CometExchange +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(30) CometNativeScan parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(31) CometFilter +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) -(35) Filter [codegen id : 11] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#43] -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [cd_demo_sk#9] +(33) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(34) CometProject +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] -(38) Project [codegen id : 11] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#44] -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#13] +(36) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +(37) CometProject +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] -(42) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#15, s_state#16] +(38) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_state#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(43) ColumnarToRow [codegen id : 9] -Input [2]: [s_store_sk#15, s_state#16] +(39) CometFilter +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#46, 2, true, false, true) = TN) AND isnotnull(s_store_sk#45)) -(44) Filter [codegen id : 9] -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +(40) CometProject +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] -(45) Project [codegen id : 9] -Output [1]: [s_store_sk#15] -Input [2]: [s_store_sk#15, s_state#16] +(41) CometBroadcastExchange +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] -(46) BroadcastExchange -Input [1]: [s_store_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(47) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 11] -Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15] - -(49) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#17, i_item_id#18] - -(50) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 11] -Output [5]: [i_item_id#18, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [7]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#17, i_item_id#18] - -(52) HashAggregate [codegen id : 11] -Input [5]: [i_item_id#18, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [1]: [i_item_id#18] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Results [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] - -(53) Exchange -Input [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 12] -Input [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -Keys [1]: [i_item_id#18] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#64, avg(UnscaledValue(agg2#20))#65, avg(UnscaledValue(agg3#21))#66, avg(UnscaledValue(agg4#22))#67] -Results [7]: [i_item_id#18, null AS s_state#68, 1 AS g_state#69, avg(agg1#19)#64 AS agg1#70, cast((avg(UnscaledValue(agg2#20))#65 / 100.0) as decimal(11,6)) AS agg2#71, cast((avg(UnscaledValue(agg3#21))#66 / 100.0) as decimal(11,6)) AS agg3#72, cast((avg(UnscaledValue(agg4#22))#67 / 100.0) as decimal(11,6)) AS agg4#73] - -(55) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(42) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight + +(43) CometProject +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(44) ReusedExchange [Reuses operator id: 24] +Output [2]: [i_item_sk#47, i_item_id#20] + +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#20] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight + +(46) CometProject +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#20] +Arguments: [i_item_id#20, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20, ss_quantity#38 AS agg1#23, ss_list_price#39 AS agg2#24, ss_coupon_amt#41 AS agg3#25, ss_sales_price#40 AS agg4#26] + +(47) CometHashAggregate +Input [5]: [i_item_id#20, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [1]: [i_item_id#20] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(48) CometExchange +Input [9]: [i_item_id#20, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(49) CometHashAggregate +Input [9]: [i_item_id#20, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] +Keys [1]: [i_item_id#20] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#63)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(56) ColumnarToRow [codegen id : 17] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(51) CometFilter +Input [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Condition : ((isnotnull(ss_cdemo_sk#57) AND isnotnull(ss_store_sk#58)) AND isnotnull(ss_item_sk#56)) -(57) Filter [codegen id : 17] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) +(52) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#64] -(58) ReusedExchange [Reuses operator id: 8] -Output [1]: [cd_demo_sk#9] +(53) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Right output [1]: [cd_demo_sk#64] +Arguments: [ss_cdemo_sk#57], [cd_demo_sk#64], Inner, BuildRight -(59) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(54) CometProject +Input [9]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63, cd_demo_sk#64] +Arguments: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63], [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] -(60) Project [codegen id : 17] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] - -(61) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#13] +(55) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#65] -(62) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(56) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Right output [1]: [d_date_sk#65] +Arguments: [ss_sold_date_sk#63], [d_date_sk#65], Inner, BuildRight -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +(57) CometProject +Input [8]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63, d_date_sk#65] +Arguments: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62], [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] -(64) ReusedExchange [Reuses operator id: 46] -Output [1]: [s_store_sk#15] +(58) ReusedExchange [Reuses operator id: 41] +Output [1]: [s_store_sk#66] -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] +Right output [1]: [s_store_sk#66] +Arguments: [ss_store_sk#58], [s_store_sk#66], Inner, BuildRight -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15] +(60) CometProject +Input [7]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, s_store_sk#66] +Arguments: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62], [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] -(67) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#17] +(61) CometNativeScan parquet spark_catalog.default.item +Output [1]: [i_item_sk#67] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(68) ColumnarToRow [codegen id : 16] -Input [1]: [i_item_sk#17] +(62) CometFilter +Input [1]: [i_item_sk#67] +Condition : isnotnull(i_item_sk#67) -(69) Filter [codegen id : 16] -Input [1]: [i_item_sk#17] -Condition : isnotnull(i_item_sk#17) +(63) CometBroadcastExchange +Input [1]: [i_item_sk#67] +Arguments: [i_item_sk#67] -(70) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(64) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] +Right output [1]: [i_item_sk#67] +Arguments: [ss_item_sk#56], [i_item_sk#67], Inner, BuildRight -(71) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(65) CometProject +Input [6]: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, i_item_sk#67] +Arguments: [agg1#23, agg2#24, agg3#25, agg4#26], [ss_quantity#59 AS agg1#23, ss_list_price#60 AS agg2#24, ss_coupon_amt#62 AS agg3#25, ss_sales_price#61 AS agg4#26] -(72) Project [codegen id : 17] -Output [4]: [ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#17] - -(73) HashAggregate [codegen id : 17] -Input [4]: [agg1#19, agg2#20, agg3#21, agg4#22] +(66) CometHashAggregate +Input [4]: [agg1#23, agg2#24, agg3#25, agg4#26] Keys: [] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#74, count#75, sum#76, count#77, sum#78, count#79, sum#80, count#81] -Results [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] -(74) Exchange -Input [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] +(67) CometExchange +Input [8]: [sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(75) HashAggregate [codegen id : 18] -Input [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +(68) CometHashAggregate +Input [8]: [sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75] Keys: [] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#90, avg(UnscaledValue(agg2#20))#91, avg(UnscaledValue(agg3#21))#92, avg(UnscaledValue(agg4#22))#93] -Results [7]: [null AS i_item_id#94, null AS s_state#95, 1 AS g_state#96, avg(agg1#19)#90 AS agg1#97, cast((avg(UnscaledValue(agg2#20))#91 / 100.0) as decimal(11,6)) AS agg2#98, cast((avg(UnscaledValue(agg3#21))#92 / 100.0) as decimal(11,6)) AS agg3#99, cast((avg(UnscaledValue(agg4#22))#93 / 100.0) as decimal(11,6)) AS agg4#100] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(69) CometUnion +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] +Child 1 Input [7]: [i_item_id#20, s_state#81, g_state#82, agg1#83, agg2#84, agg3#85, agg4#86] +Child 2 Input [7]: [i_item_id#87, s_state#88, g_state#89, agg1#90, agg2#91, agg3#92, agg4#93] -(76) Union +(70) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#76,agg1#77,agg2#78,agg3#79,agg4#80]), [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] -(77) TakeOrderedAndProject -Input [7]: [i_item_id#18, s_state#16, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] -Arguments: 100, [i_item_id#18 ASC NULLS FIRST, s_state#16 ASC NULLS FIRST], [i_item_id#18, s_state#16, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] +(71) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/extended.txt new file mode 100644 index 0000000000..180236b925 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/extended.txt @@ -0,0 +1,91 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 88 out of 88 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/simplified.txt index fd5fca8f6a..a7b0657337 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/simplified.txt @@ -1,113 +1,73 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #6 - WholeStageCodegen (11) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange #8 - WholeStageCodegen (17) - HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [s_store_sk] #7 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (16) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id,s_state] #1 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id] #6 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [s_store_sk] #7 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange #8 + CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #7 + CometBroadcastExchange [i_item_sk] #9 + CometFilter [i_item_sk] + CometNativeScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/explain.txt index d0eba50a09..7e284f613e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/explain.txt @@ -1,208 +1,189 @@ == Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometNativeScan parquet spark_catalog.default.customer (25) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +(8) CometProject Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#9, s_county#10] Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] +(11) CometProject Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#9] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +(14) CometProject Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometNativeScan parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] +(17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#11] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] +(20) CometProject Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 15) AND (cnt#18 <= 20)) +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(25) CometNativeScan parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST], true, 0 +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/extended.txt new file mode 100644 index 0000000000..7eeb1f79c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/extended.txt @@ -0,0 +1,35 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer + +Comet accelerated 32 out of 32 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/simplified.txt index 67af698912..cb3f282260 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/simplified.txt @@ -1,54 +1,35 @@ -WholeStageCodegen (7) - Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/explain.txt index 23a0bff984..08a13c556e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/explain.txt @@ -1,267 +1,269 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometNativeScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (36) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] +(6) CometProject Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] +(9) CometProject Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#6] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(20) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#13] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None -(29) Filter [codegen id : 9] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(30) Project [codegen id : 9] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] +(29) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] +(30) CometFilter +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : isnotnull(ca_address_sk#15) -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) +(31) CometProject +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15, ca_state#17], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#17] -(34) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#15, ca_state#17] -(35) BroadcastHashJoin [codegen id : 9] +(33) BroadcastExchange +Input [2]: [ca_address_sk#15, ca_state#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(36) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#17] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#17] -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(37) CometFilter +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Condition : isnotnull(cd_demo_sk#18) + +(38) CometProject +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (40) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(41) BroadcastHashJoin [codegen id : 9] +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#18] Join type: Inner Join condition: None -(42) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(43) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] - -(44) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, avg(cd_dep_count#22)#52, max(cd_dep_count#22)#53, sum(cd_dep_count#22)#54, avg(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, sum(cd_dep_employed_count#23)#57, avg(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, sum(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, count(1)#51 AS cnt1#61, avg(cd_dep_count#22)#52 AS avg(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, sum(cd_dep_count#22)#54 AS sum(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, avg(cd_dep_employed_count#23)#55 AS avg(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, sum(cd_dep_employed_count#23)#57 AS sum(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, avg(cd_dep_college_count#24)#58 AS avg(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, sum(cd_dep_college_count#24)#60 AS sum(cd_dep_college_count)#72] - -(46) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] +(42) Project [codegen id : 5] +Output [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Input [8]: [c_current_cdemo_sk#4, ca_state#17, cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#21), partial_max(cd_dep_count#21), partial_sum(cd_dep_count#21), partial_avg(cd_dep_employed_count#22), partial_max(cd_dep_employed_count#22), partial_sum(cd_dep_employed_count#22), partial_avg(cd_dep_college_count#23), partial_max(cd_dep_college_count#23), partial_sum(cd_dep_college_count#23)] +Aggregate Attributes [13]: [count#26, sum#27, count#28, max#29, sum#30, sum#31, count#32, max#33, sum#34, sum#35, count#36, max#37, sum#38] +Results [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] + +(44) CometColumnarExchange +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] +Arguments: hashpartitioning(ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [count(1), avg(cd_dep_count#21), max(cd_dep_count#21), sum(cd_dep_count#21), avg(cd_dep_employed_count#22), max(cd_dep_employed_count#22), sum(cd_dep_employed_count#22), avg(cd_dep_college_count#23), max(cd_dep_college_count#23), sum(cd_dep_college_count#23)] +Aggregate Attributes [10]: [count(1)#52, avg(cd_dep_count#21)#53, max(cd_dep_count#21)#54, sum(cd_dep_count#21)#55, avg(cd_dep_employed_count#22)#56, max(cd_dep_employed_count#22)#57, sum(cd_dep_employed_count#22)#58, avg(cd_dep_college_count#23)#59, max(cd_dep_college_count#23)#60, sum(cd_dep_college_count#23)#61] +Results [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, count(1)#52 AS cnt1#62, avg(cd_dep_count#21)#53 AS avg(cd_dep_count)#63, max(cd_dep_count#21)#54 AS max(cd_dep_count)#64, sum(cd_dep_count#21)#55 AS sum(cd_dep_count)#65, cd_dep_employed_count#22, count(1)#52 AS cnt2#66, avg(cd_dep_employed_count#22)#56 AS avg(cd_dep_employed_count)#67, max(cd_dep_employed_count#22)#57 AS max(cd_dep_employed_count)#68, sum(cd_dep_employed_count#22)#58 AS sum(cd_dep_employed_count)#69, cd_dep_college_count#23, count(1)#52 AS cnt3#70, avg(cd_dep_college_count#23)#59 AS avg(cd_dep_college_count)#71, max(cd_dep_college_count#23)#60 AS max(cd_dep_college_count)#72, sum(cd_dep_college_count#23)#61 AS sum(cd_dep_college_count)#73] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cnt1#62, avg(cd_dep_count)#63, max(cd_dep_count)#64, sum(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, avg(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, sum(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, avg(cd_dep_college_count)#71, max(cd_dep_college_count)#72, sum(cd_dep_college_count)#73] +Arguments: 100, [ca_state#17 ASC NULLS FIRST, cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_dep_count#21 ASC NULLS FIRST, cd_dep_employed_count#22 ASC NULLS FIRST, cd_dep_college_count#23 ASC NULLS FIRST], [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cnt1#62, avg(cd_dep_count)#63, max(cd_dep_count)#64, sum(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, avg(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, sum(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, avg(cd_dep_college_count)#71, max(cd_dep_college_count)#72, sum(cd_dep_college_count)#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/extended.txt new file mode 100644 index 0000000000..4c223685f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/extended.txt @@ -0,0 +1,55 @@ + TakeOrderedAndProject [COMET: ] ++- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- BroadcastHashJoin + : : :- BroadcastHashJoin + : : : :- CometColumnarToRow + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- CometColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- BroadcastExchange + +- CometColumnarToRow + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 32 out of 47 eligible operators (68%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/simplified.txt index 46a3b243ff..4b9be3b971 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/simplified.txt @@ -1,70 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/explain.txt index 33b8893391..524af25843 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/explain.txt @@ -1,253 +1,230 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * BroadcastHashJoin LeftSemi BuildRight (27) - : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : +- BroadcastExchange (13) - : : : +- * Project (12) - : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : :- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : +- BroadcastExchange (26) - : : +- Union (25) - : : :- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (22) - : +- BroadcastExchange (32) - : +- * Filter (31) - : +- * ColumnarToRow (30) - : +- Scan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * Filter (37) - +- * ColumnarToRow (36) - +- Scan parquet spark_catalog.default.customer_demographics (35) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometNativeScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometNativeScan parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometNativeScan parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometNativeScan parquet spark_catalog.default.customer_demographics (30) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometNativeScan parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(6) CometProject Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] +(9) CometProject Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#4] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometNativeScan parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9 AS customsk#12] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [customsk#11], [ws_bill_customer_sk#9 AS customsk#11] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +(16) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(22) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#15] +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#14] -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#13 AS customsk#16] -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +(19) CometProject +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [customsk#15], [cs_ship_customer_sk#12 AS customsk#15] -(25) Union +(20) CometUnion +Child 0 Input [1]: [customsk#11] +Child 1 Input [1]: [customsk#15] -(26) BroadcastExchange -Input [1]: [customsk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(21) CometBroadcastExchange +Input [1]: [customsk#11] +Arguments: [customsk#11] -(27) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [customsk#12] -Join type: LeftSemi -Join condition: None +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customsk#11] +Arguments: [c_customer_sk#1], [customsk#11], LeftSemi, BuildRight -(28) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +(23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(29) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] +(24) CometNativeScan parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] +(25) CometFilter +Input [2]: [ca_address_sk#16, ca_state#17] +Condition : isnotnull(ca_address_sk#16) -(31) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) +(26) CometProject +Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16, ca_state#18], [ca_address_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) AS ca_state#18] -(32) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(27) CometBroadcastExchange +Input [2]: [ca_address_sk#16, ca_state#18] +Arguments: [ca_address_sk#16, ca_state#18] -(33) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [2]: [ca_address_sk#16, ca_state#18] +Arguments: [c_current_addr_sk#3], [ca_address_sk#16], Inner, BuildRight -(34) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, ca_state#18] -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#18] +(29) CometProject +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16, ca_state#18] +Arguments: [c_current_cdemo_sk#2, ca_state#18], [c_current_cdemo_sk#2, ca_state#18] -(35) Scan parquet spark_catalog.default.customer_demographics +(30) CometNativeScan parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(37) Filter [codegen id : 8] +(31) CometFilter Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Condition : isnotnull(cd_demo_sk#19) -(38) BroadcastExchange +(32) CometProject Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#20, 1, true, false, true) AS cd_gender#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] + +(33) CometBroadcastExchange +Input [6]: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, ca_state#18] +Right output [6]: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight -(40) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(35) CometProject +Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -(41) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometHashAggregate +Input [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Keys [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -(42) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(37) CometExchange +Input [19]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39] +Arguments: hashpartitioning(ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(43) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(38) CometHashAggregate +Input [19]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39] +Keys [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, avg(cd_dep_count#22)#52, max(cd_dep_count#22)#53, sum(cd_dep_count#22)#54, avg(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, sum(cd_dep_employed_count#23)#57, avg(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, sum(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, count(1)#51 AS cnt1#61, avg(cd_dep_count#22)#52 AS avg(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, sum(cd_dep_count#22)#54 AS sum(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, avg(cd_dep_employed_count#23)#55 AS avg(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, sum(cd_dep_employed_count#23)#57 AS sum(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, avg(cd_dep_college_count#24)#58 AS avg(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, sum(cd_dep_college_count#24)#60 AS sum(cd_dep_college_count)#72] -(44) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] +(39) CometTakeOrderedAndProject +Input [18]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#18 ASC NULLS FIRST,cd_gender#25 ASC NULLS FIRST,cd_marital_status#26 ASC NULLS FIRST,cd_dep_count#22 ASC NULLS FIRST,cd_dep_employed_count#23 ASC NULLS FIRST,cd_dep_college_count#24 ASC NULLS FIRST], output=[ca_state#18,cd_gender#25,cd_marital_status#26,cd_dep_count#22,cnt1#40,avg(cd_dep_count)#41,max(cd_dep_count)#42,sum(cd_dep_count)#43,cd_dep_employed_count#23,cnt2#44,avg(cd_dep_employed_count)#45,max(cd_dep_employed_count)#46,sum(cd_dep_employed_count)#47,cd_dep_college_count#24,cnt3#48,avg(cd_dep_college_count)#49,max(cd_dep_college_count)#50,sum(cd_dep_college_count)#51]), [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51], 100, 0, [ca_state#18 ASC NULLS FIRST, cd_gender#25 ASC NULLS FIRST, cd_marital_status#26 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] + +(40) CometColumnarToRow [codegen id : 1] +Input [18]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/extended.txt new file mode 100644 index 0000000000..42c478c7c8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/extended.txt @@ -0,0 +1,48 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometUnion + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.customer_address + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer_demographics + +Comet accelerated 45 out of 45 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/simplified.txt index 11b6f555e2..86604080fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/simplified.txt @@ -1,67 +1,42 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - WholeStageCodegen (10) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,customsk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - Union - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometHashAggregate [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk,ca_state] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk,ca_state] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [customsk] #4 + CometUnion [customsk] + CometProject [ws_bill_customer_sk] [customsk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customsk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/explain.txt index e464aa75d3..938e1f4bc8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/explain.txt @@ -1,269 +1,242 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- Union (37) - :- * HashAggregate (26) - : +- Exchange (25) - : +- * HashAggregate (24) - : +- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.item (11) - : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - :- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * HashAggregate (28) - : +- ReusedExchange (27) - +- * HashAggregate (36) - +- Exchange (35) - +- * HashAggregate (34) - +- * HashAggregate (33) - +- ReusedExchange (32) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (43) ++- * Project (42) + +- Window (41) + +- * CometColumnarToRow (40) + +- CometSort (39) + +- CometExchange (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometUnion (34) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.store (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometHashAggregate (30) + +- ReusedExchange (29) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#6, d_year#7] Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +(8) CometProject Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#8] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] -(16) Project [codegen id : 4] -Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometNativeScan parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] - -(19) Filter [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) -(20) Project [codegen id : 3] -Output [1]: [s_store_sk#11] -Input [2]: [s_store_sk#11, s_state#12] +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(21) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight -(23) Project [codegen id : 4] -Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] -(24) HashAggregate [codegen id : 4] -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Keys [2]: [i_category#10, i_class#9] +(21) CometHashAggregate +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#13, sum#14] -Results [4]: [i_category#10, i_class#9, sum#15, sum#16] -(25) Exchange -Input [4]: [i_category#10, i_class#9, sum#15, sum#16] -Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [4]: [i_category#10, i_class#9, sum#15, sum#16] -Keys [2]: [i_category#10, i_class#9] +(23) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#17, sum(UnscaledValue(ss_ext_sales_price#3))#18] -Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#17,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2)) as decimal(38,20)) AS gross_margin#19, i_category#10, i_class#9, 0 AS t_category#20, 0 AS t_class#21, 0 AS lochierarchy#22] -(27) ReusedExchange [Reuses operator id: 25] -Output [4]: [i_category#10, i_class#9, sum#23, sum#24] +(24) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#17, sum#18] -(28) HashAggregate [codegen id : 10] -Input [4]: [i_category#10, i_class#9, sum#23, sum#24] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26] -Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2) AS ss_net_profit#27, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2) AS ss_ext_sales_price#28, i_category#10] - -(29) HashAggregate [codegen id : 10] -Input [3]: [ss_net_profit#27, ss_ext_sales_price#28, i_category#10] -Keys [1]: [i_category#10] -Functions [2]: [partial_sum(ss_net_profit#27), partial_sum(ss_ext_sales_price#28)] -Aggregate Attributes [4]: [sum#29, isEmpty#30, sum#31, isEmpty#32] -Results [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] - -(30) Exchange -Input [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] -Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) HashAggregate [codegen id : 11] -Input [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] -Keys [1]: [i_category#10] -Functions [2]: [sum(ss_net_profit#27), sum(ss_ext_sales_price#28)] -Aggregate Attributes [2]: [sum(ss_net_profit#27)#37, sum(ss_ext_sales_price#28)#38] -Results [6]: [cast((sum(ss_net_profit#27)#37 / sum(ss_ext_sales_price#28)#38) as decimal(38,20)) AS gross_margin#39, i_category#10, null AS i_class#40, 0 AS t_category#41, 1 AS t_class#42, 1 AS lochierarchy#43] - -(32) ReusedExchange [Reuses operator id: 25] -Output [4]: [i_category#10, i_class#9, sum#44, sum#45] - -(33) HashAggregate [codegen id : 16] -Input [4]: [i_category#10, i_class#9, sum#44, sum#45] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2) AS ss_net_profit#27, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2) AS ss_ext_sales_price#28] +(25) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#17, sum#18] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#19)), sum(UnscaledValue(ss_ext_sales_price#20))] + +(26) CometHashAggregate +Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#12] +Keys [1]: [i_category#12] +Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] + +(27) CometExchange +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(i_category#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) HashAggregate [codegen id : 16] -Input [2]: [ss_net_profit#27, ss_ext_sales_price#28] +(28) CometHashAggregate +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [i_category#12] +Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] + +(29) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#27, sum#28] + +(30) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#27, sum#28] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] + +(31) CometHashAggregate +Input [2]: [ss_net_profit#21, ss_ext_sales_price#22] Keys: [] -Functions [2]: [partial_sum(ss_net_profit#27), partial_sum(ss_ext_sales_price#28)] -Aggregate Attributes [4]: [sum#46, isEmpty#47, sum#48, isEmpty#49] -Results [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] +Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] -(35) Exchange -Input [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometExchange +Input [4]: [sum#31, isEmpty#32, sum#33, isEmpty#34] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(36) HashAggregate [codegen id : 17] -Input [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] +(33) CometHashAggregate +Input [4]: [sum#31, isEmpty#32, sum#33, isEmpty#34] Keys: [] -Functions [2]: [sum(ss_net_profit#27), sum(ss_ext_sales_price#28)] -Aggregate Attributes [2]: [sum(ss_net_profit#27)#54, sum(ss_ext_sales_price#28)#55] -Results [6]: [cast((sum(ss_net_profit#27)#54 / sum(ss_ext_sales_price#28)#55) as decimal(38,20)) AS gross_margin#56, null AS i_category#57, null AS i_class#58, 1 AS t_category#59, 1 AS t_class#60, 2 AS lochierarchy#61] +Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] -(37) Union +(34) CometUnion +Child 0 Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Child 1 Input [6]: [gross_margin#41, i_category#12, i_class#42, t_category#43, t_class#44, lochierarchy#45] +Child 2 Input [6]: [gross_margin#46, i_category#47, i_class#48, t_category#49, t_class#50, lochierarchy#51] -(38) HashAggregate [codegen id : 18] -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Keys [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] +(35) CometHashAggregate +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Keys [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] Functions: [] -Aggregate Attributes: [] -Results [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -(39) Exchange -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Arguments: hashpartitioning(gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(36) CometExchange +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Arguments: hashpartitioning(gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 19] -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Keys [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] +(37) CometHashAggregate +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Keys [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] Functions: [] -Aggregate Attributes: [] -Results [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, CASE WHEN (t_class#21 = 0) THEN i_category#10 END AS _w0#62] -(41) Exchange -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: hashpartitioning(lochierarchy#22, _w0#62, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(38) CometExchange +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: hashpartitioning(lochierarchy#40, _w0#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(39) CometSort +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52], [lochierarchy#40 ASC NULLS FIRST, _w0#52 ASC NULLS FIRST, gross_margin#35 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: [lochierarchy#22 ASC NULLS FIRST, _w0#62 ASC NULLS FIRST, gross_margin#19 ASC NULLS FIRST], false, 0 +(40) CometColumnarToRow [codegen id : 1] +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] -(43) Window -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: [rank(gross_margin#19) windowspecdefinition(lochierarchy#22, _w0#62, gross_margin#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#63], [lochierarchy#22, _w0#62], [gross_margin#19 ASC NULLS FIRST] +(41) Window +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: [rank(gross_margin#35) windowspecdefinition(lochierarchy#40, _w0#52, gross_margin#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#53], [lochierarchy#40, _w0#52], [gross_margin#35 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] -Input [6]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62, rank_within_parent#63] +(42) Project [codegen id : 2] +Output [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] +Input [6]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52, rank_within_parent#53] -(45) TakeOrderedAndProject -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] -Arguments: 100, [lochierarchy#22 DESC NULLS LAST, CASE WHEN (lochierarchy#22 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#63 ASC NULLS FIRST], [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] +(43) TakeOrderedAndProject +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] +Arguments: 100, [lochierarchy#40 DESC NULLS LAST, CASE WHEN (lochierarchy#40 = 0) THEN i_category#36 END ASC NULLS FIRST, rank_within_parent#53 ASC NULLS FIRST], [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/extended.txt new file mode 100644 index 0000000000..30d5d43026 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/extended.txt @@ -0,0 +1,87 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 81 out of 84 eligible operators (96%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/simplified.txt index ad136a38ae..62746a00f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/simplified.txt @@ -1,74 +1,47 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (21) + WholeStageCodegen (2) Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [gross_margin,lochierarchy,_w0] - WholeStageCodegen (20) - Sort [lochierarchy,_w0,gross_margin] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (19) - HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] [_w0] - InputAdapter - Exchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 - WholeStageCodegen (18) - HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,t_category,t_class,lochierarchy,sum,sum] - InputAdapter - Exchange [i_category,i_class] #3 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - WholeStageCodegen (11) - HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [i_category] #7 - WholeStageCodegen (10) - HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #8 - WholeStageCodegen (16) - HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [t_category] [gross_margin,i_category,i_class,lochierarchy,_w0,t_class] + CometExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 + CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + CometUnion [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),i_category,i_class] + CometExchange [i_category,i_class] #3 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #6 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometExchange [i_category] #7 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,i_category,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + ReusedExchange [i_category,i_class,sum,sum] #3 + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometExchange #8 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_category,i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/explain.txt index 4a4b596795..50eea35380 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.store (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#4] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(14) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) -(18) Filter [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] -(19) BroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#5] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] -(21) Project [codegen id : 4] -Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] -(27) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] -(30) Filter [codegen id : 22] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -(32) ReusedExchange [Reuses operator id: 23] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(33) HashAggregate [codegen id : 12] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#17] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(36) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#17 AS sum_sales#30, rn#29] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17, rn#29] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(38) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#29 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] + +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(41) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#17 AS sum_sales#38, rn#37] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17, rn#37] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=8] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#37 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] +(47) Project [codegen id : 7] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/extended.txt new file mode 100644 index 0000000000..eca90ab6dc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.store + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/simplified.txt index 32992e11f2..4f03ba59f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/explain.txt index 5548f821e9..cba2ffa7be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/explain.txt @@ -1,457 +1,425 @@ == Physical Plan == -TakeOrderedAndProject (81) -+- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- Union (77) - :- * Project (28) - : +- * Filter (27) - : +- Window (26) - : +- * Sort (25) - : +- Window (24) - : +- * Sort (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- Scan parquet spark_catalog.default.web_returns (6) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - :- * Project (52) - : +- * Filter (51) - : +- Window (50) - : +- * Sort (49) - : +- Window (48) - : +- * Sort (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildLeft (38) - : : :- BroadcastExchange (33) - : : : +- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.catalog_sales (29) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.catalog_returns (34) - : +- ReusedExchange (40) - +- * Project (76) - +- * Filter (75) - +- Window (74) - +- * Sort (73) - +- Window (72) - +- * Sort (71) - +- Exchange (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildLeft (62) - : :- BroadcastExchange (57) - : : +- * Project (56) - : : +- * Filter (55) - : : +- * ColumnarToRow (54) - : : +- Scan parquet spark_catalog.default.store_sales (53) - : +- * Project (61) - : +- * Filter (60) - : +- * ColumnarToRow (59) - : +- Scan parquet spark_catalog.default.store_returns (58) - +- ReusedExchange (64) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometNativeScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometNativeScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +(3) CometProject Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) BroadcastExchange +(4) CometBroadcastExchange Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(6) Scan parquet spark_catalog.default.web_returns +(5) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(7) ColumnarToRow -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(8) Filter +(6) CometFilter Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) -(9) Project -Output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +(7) CometProject Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#8, wr_item_sk#7] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +(9) CometProject Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(12) Scan parquet spark_catalog.default.date_dim +(10) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_moy#14] Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#12] +(12) CometProject Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12] -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#6] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight -(18) Project [codegen id : 3] -Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] +(15) CometProject Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -(19) HashAggregate [codegen id : 3] +(16) CometHashAggregate Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(20) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 4] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] -(22) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(23) Sort [codegen id : 5] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(24) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] -(25) Sort [codegen id : 6] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 +(22) Window +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] -(26) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 -(27) Filter [codegen id : 7] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) +(24) Window +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) -(28) Project [codegen id : 7] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(29) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(27) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(31) Filter [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : (((((((isnotnull(cs_net_profit#41) AND isnotnull(cs_net_paid#40)) AND isnotnull(cs_quantity#39)) AND (cs_net_profit#41 > 1.00)) AND (cs_net_paid#40 > 0.00)) AND (cs_quantity#39 > 0)) AND isnotnull(cs_order_number#38)) AND isnotnull(cs_item_sk#37)) +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) -(32) Project [codegen id : 8] -Output [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(33) BroadcastExchange -Input [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=5] +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(34) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] +(31) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(35) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(36) Filter -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) - -(37) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#38, cs_item_sk#37] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [6]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46] -Input [9]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] - -(40) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#48] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#42] -Right keys [1]: [d_date_sk#48] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Input [7]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] - -(43) HashAggregate [codegen id : 10] -Input [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Keys [1]: [cs_item_sk#37] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#39, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Results [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] - -(44) Exchange -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Keys [1]: [cs_item_sk#37] -Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#39, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#39, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64] -Results [3]: [cs_item_sk#37 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#39, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] - -(46) Exchange -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(47) Sort [codegen id : 12] -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 - -(48) Window -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] - -(49) Sort [codegen id : 13] -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 - -(50) Window -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] - -(51) Filter [codegen id : 14] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) - -(52) Project [codegen id : 14] -Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] - -(53) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometNativeScan parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(55) Filter [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) -(56) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(57) BroadcastExchange -Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=8] +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(58) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] +(54) CometNativeScan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(59) ColumnarToRow -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(60) Filter -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Condition : (((isnotnull(sr_return_amt#80) AND (sr_return_amt#80 > 10000.00)) AND isnotnull(sr_ticket_number#78)) AND isnotnull(sr_item_sk#77)) - -(61) Project -Output [4]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(62) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#78, sr_item_sk#77] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80] -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] - -(64) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#82] - -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#76] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80, d_date_sk#82] - -(67) HashAggregate [codegen id : 17] -Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Keys [1]: [ss_item_sk#71] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#79, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#83, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] -Results [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] - -(68) Exchange -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(69) HashAggregate [codegen id : 18] -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Keys [1]: [ss_item_sk#71] -Functions [4]: [sum(coalesce(sr_return_quantity#79, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#79, 0))#95, sum(coalesce(ss_quantity#73, 0))#96, sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98] -Results [3]: [ss_item_sk#71 AS item#99, (cast(sum(coalesce(sr_return_quantity#79, 0))#95 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#96 as decimal(15,4))) AS return_ratio#100, (cast(sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98 as decimal(15,4))) AS currency_ratio#101] - -(70) Exchange -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(71) Sort [codegen id : 19] -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [return_ratio#100 ASC NULLS FIRST], false, 0 - -(72) Window -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [rank(return_ratio#100) windowspecdefinition(return_ratio#100 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#102], [return_ratio#100 ASC NULLS FIRST] - -(73) Sort [codegen id : 20] -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [currency_ratio#101 ASC NULLS FIRST], false, 0 - -(74) Window -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [rank(currency_ratio#101) windowspecdefinition(currency_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#103], [currency_ratio#101 ASC NULLS FIRST] - -(75) Filter [codegen id : 21] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] -Condition : ((return_rank#102 <= 10) OR (currency_rank#103 <= 10)) - -(76) Project [codegen id : 21] -Output [5]: [store AS channel#104, item#99, return_ratio#100, return_rank#102, currency_rank#103] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] - -(77) Union - -(78) HashAggregate [codegen id : 22] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(79) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(80) HashAggregate [codegen id : 23] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -(81) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST, item#31 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/extended.txt new file mode 100644 index 0000000000..262571ed59 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/extended.txt @@ -0,0 +1,86 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- Project + : +- Filter + : +- Window + : +- Sort + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- Project + +- Filter + +- Window + +- Sort + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 63 out of 80 eligible operators (78%). Final plan contains 4 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/simplified.txt index 018748b274..117b74a75b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/simplified.txt @@ -1,129 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (7) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (6) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (5) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (4) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (14) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (13) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (12) - Sort [return_ratio] - InputAdapter - Exchange #6 - WholeStageCodegen (11) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #7 - WholeStageCodegen (10) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (20) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (19) - Sort [return_ratio] - InputAdapter - Exchange #9 - WholeStageCodegen (18) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (17) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/explain.txt index 40f1f3e69c..e9fc524fbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/explain.txt @@ -1,404 +1,413 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- * HashAggregate (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- Window (60) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * SortMergeJoin FullOuter (55) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- Exchange (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (17) - : : : +- Window (16) - : : : +- * Sort (15) - : : : +- Exchange (14) - : : : +- * HashAggregate (13) - : : : +- Exchange (12) - : : : +- * HashAggregate (11) - : : : +- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * Sort (19) - : : +- ReusedExchange (18) - : +- * Sort (54) - : +- Exchange (53) - : +- * HashAggregate (52) - : +- Exchange (51) - : +- * HashAggregate (50) - : +- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- Window (41) - : : +- * Sort (40) - : : +- Exchange (39) - : : +- * HashAggregate (38) - : : +- Exchange (37) - : : +- * HashAggregate (36) - : : +- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Filter (32) - : : : +- * ColumnarToRow (31) - : : : +- Scan parquet spark_catalog.default.store_sales (30) - : : +- ReusedExchange (33) - : +- BroadcastExchange (47) - : +- * Project (46) - : +- Window (45) - : +- * Sort (44) - : +- ReusedExchange (43) - +- BroadcastExchange (65) - +- * Project (64) - +- Window (63) - +- * Sort (62) - +- ReusedExchange (61) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (76) ++- * Filter (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- Window (64) + : +- * CometColumnarToRow (63) + : +- CometSort (62) + : +- CometExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometSortMergeJoin (58) + : :- CometSort (30) + : : +- CometColumnarExchange (29) + : : +- * HashAggregate (28) + : : +- * CometColumnarToRow (27) + : : +- CometColumnarExchange (26) + : : +- * HashAggregate (25) + : : +- * Project (24) + : : +- * BroadcastHashJoin Inner BuildRight (23) + : : :- * Project (16) + : : : +- Window (15) + : : : +- * CometColumnarToRow (14) + : : : +- CometSort (13) + : : : +- CometExchange (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (22) + : : +- * Project (21) + : : +- Window (20) + : : +- * CometColumnarToRow (19) + : : +- CometSort (18) + : : +- ReusedExchange (17) + : +- CometSort (57) + : +- CometColumnarExchange (56) + : +- * HashAggregate (55) + : +- * CometColumnarToRow (54) + : +- CometColumnarExchange (53) + : +- * HashAggregate (52) + : +- * Project (51) + : +- * BroadcastHashJoin Inner BuildRight (50) + : :- * Project (43) + : : +- Window (42) + : : +- * CometColumnarToRow (41) + : : +- CometSort (40) + : : +- CometExchange (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometFilter (32) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (31) + : : +- ReusedExchange (33) + : +- BroadcastExchange (49) + : +- * Project (48) + : +- Window (47) + : +- * CometColumnarToRow (46) + : +- CometSort (45) + : +- ReusedExchange (44) + +- BroadcastExchange (70) + +- * Project (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- ReusedExchange (65) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] +(5) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_date#5] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 2] -Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +(8) CometProject Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] -(11) HashAggregate [codegen id : 2] +(9) CometHashAggregate Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] -(12) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) HashAggregate [codegen id : 3] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] +(11) CometHashAggregate +Input [3]: [ws_item_sk#1, d_date#5, sum#7] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS sumws#11, ws_item_sk#1] -(14) Exchange -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(12) CometExchange +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(13) CometSort +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(15) Sort [codegen id : 4] -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +(14) CometColumnarToRow [codegen id : 1] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] -(16) Window -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] +(15) Window +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] -(17) Project [codegen id : 10] -Output [4]: [item_sk#10, d_date#5, sumws#11, rk#12] -Input [5]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1, rk#12] +(16) Project [codegen id : 4] +Output [4]: [item_sk#8, d_date#5, sumws#9, rk#10] +Input [5]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1, rk#10] -(18) ReusedExchange [Reuses operator id: 14] -Output [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] +(17) ReusedExchange [Reuses operator id: 12] +Output [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] -(19) Sort [codegen id : 8] -Input [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] -Arguments: [ws_item_sk#14 ASC NULLS FIRST, d_date#13 ASC NULLS FIRST], false, 0 +(18) CometSort +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12], [ws_item_sk#12 ASC NULLS FIRST, d_date#11 ASC NULLS FIRST] + +(19) CometColumnarToRow [codegen id : 2] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (20) Window -Input [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] -Arguments: [row_number() windowspecdefinition(ws_item_sk#14, d_date#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#15], [ws_item_sk#14], [d_date#13 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [row_number() windowspecdefinition(ws_item_sk#12, d_date#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#13], [ws_item_sk#12], [d_date#11 ASC NULLS FIRST] -(21) Project [codegen id : 9] -Output [3]: [item_sk#10 AS item_sk#16, sumws#11 AS sumws#17, rk#15] -Input [5]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14, rk#15] +(21) Project [codegen id : 3] +Output [3]: [item_sk#8 AS item_sk#14, sumws#9 AS sumws#15, rk#13] +Input [5]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12, rk#13] (22) BroadcastExchange -Input [3]: [item_sk#16, sumws#17, rk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Input [3]: [item_sk#14, sumws#15, rk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(23) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [item_sk#10] -Right keys [1]: [item_sk#16] +(23) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [item_sk#8] +Right keys [1]: [item_sk#14] Join type: Inner -Join condition: (rk#12 >= rk#15) - -(24) Project [codegen id : 10] -Output [4]: [item_sk#10, d_date#5, sumws#11, sumws#17] -Input [7]: [item_sk#10, d_date#5, sumws#11, rk#12, item_sk#16, sumws#17, rk#15] - -(25) HashAggregate [codegen id : 10] -Input [4]: [item_sk#10, d_date#5, sumws#11, sumws#17] -Keys [3]: [item_sk#10, d_date#5, sumws#11] -Functions [1]: [partial_sum(sumws#17)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] - -(26) Exchange -Input [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] -Arguments: hashpartitioning(item_sk#10, d_date#5, sumws#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(27) HashAggregate [codegen id : 11] -Input [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] -Keys [3]: [item_sk#10, d_date#5, sumws#11] -Functions [1]: [sum(sumws#17)] -Aggregate Attributes [1]: [sum(sumws#17)#22] -Results [3]: [item_sk#10, d_date#5, sum(sumws#17)#22 AS cume_sales#23] - -(28) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#23] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(29) Sort [codegen id : 12] -Input [3]: [item_sk#10, d_date#5, cume_sales#23] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] +Join condition: (rk#10 >= rk#13) + +(24) Project [codegen id : 4] +Output [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Input [7]: [item_sk#8, d_date#5, sumws#9, rk#10, item_sk#14, sumws#15, rk#13] + +(25) HashAggregate [codegen id : 4] +Input [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [partial_sum(sumws#15)] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] + +(26) CometColumnarExchange +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(27) CometColumnarToRow [codegen id : 5] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] + +(28) HashAggregate [codegen id : 5] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [sum(sumws#15)] +Aggregate Attributes [1]: [sum(sumws#15)#20] +Results [3]: [item_sk#8, d_date#5, sum(sumws#15)#20 AS cume_sales#21] + +(29) CometColumnarExchange +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(30) CometSort +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: [item_sk#8, d_date#5, cume_sales#21], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(31) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] +(32) CometFilter +Input [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_item_sk#22) -(32) Filter [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#24) +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#25, d_date#26] -(33) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#27, d_date#28] +(34) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Right output [2]: [d_date_sk#25, d_date#26] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 14] -Output [3]: [ss_item_sk#24, ss_sales_price#25, d_date#28] -Input [5]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_date#28] - -(36) HashAggregate [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, d_date#28] -Keys [2]: [ss_item_sk#24, d_date#28] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#25))] -Aggregate Attributes [1]: [sum#29] -Results [3]: [ss_item_sk#24, d_date#28, sum#30] - -(37) Exchange -Input [3]: [ss_item_sk#24, d_date#28, sum#30] -Arguments: hashpartitioning(ss_item_sk#24, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(38) HashAggregate [codegen id : 15] -Input [3]: [ss_item_sk#24, d_date#28, sum#30] -Keys [2]: [ss_item_sk#24, d_date#28] -Functions [1]: [sum(UnscaledValue(ss_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#25))#31] -Results [4]: [ss_item_sk#24 AS item_sk#32, d_date#28, MakeDecimal(sum(UnscaledValue(ss_sales_price#25))#31,17,2) AS sumss#33, ss_item_sk#24] - -(39) Exchange -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: hashpartitioning(ss_item_sk#24, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(40) Sort [codegen id : 16] -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: [ss_item_sk#24 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 - -(41) Window -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: [row_number() windowspecdefinition(ss_item_sk#24, d_date#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#34], [ss_item_sk#24], [d_date#28 ASC NULLS FIRST] - -(42) Project [codegen id : 22] -Output [4]: [item_sk#32, d_date#28, sumss#33, rk#34] -Input [5]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24, rk#34] - -(43) ReusedExchange [Reuses operator id: 39] -Output [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] - -(44) Sort [codegen id : 20] -Input [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] -Arguments: [ss_item_sk#36 ASC NULLS FIRST, d_date#35 ASC NULLS FIRST], false, 0 - -(45) Window -Input [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] -Arguments: [row_number() windowspecdefinition(ss_item_sk#36, d_date#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#37], [ss_item_sk#36], [d_date#35 ASC NULLS FIRST] - -(46) Project [codegen id : 21] -Output [3]: [item_sk#32 AS item_sk#38, sumss#33 AS sumss#39, rk#37] -Input [5]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36, rk#37] - -(47) BroadcastExchange -Input [3]: [item_sk#38, sumss#39, rk#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -(48) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [item_sk#32] -Right keys [1]: [item_sk#38] +(35) CometProject +Input [5]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_date#26] +Arguments: [ss_item_sk#22, ss_sales_price#23, d_date#26], [ss_item_sk#22, ss_sales_price#23, d_date#26] + +(36) CometHashAggregate +Input [3]: [ss_item_sk#22, ss_sales_price#23, d_date#26] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#23))] + +(37) CometExchange +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Arguments: hashpartitioning(ss_item_sk#22, d_date#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(38) CometHashAggregate +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [sum(UnscaledValue(ss_sales_price#23))] + +(39) CometExchange +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: hashpartitioning(ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(40) CometSort +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22], [ss_item_sk#22 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] + +(41) CometColumnarToRow [codegen id : 6] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] + +(42) Window +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [row_number() windowspecdefinition(ss_item_sk#22, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [ss_item_sk#22], [d_date#26 ASC NULLS FIRST] + +(43) Project [codegen id : 9] +Output [4]: [item_sk#28, d_date#26, sumss#29, rk#30] +Input [5]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22, rk#30] + +(44) ReusedExchange [Reuses operator id: 39] +Output [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] + +(45) CometSort +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32], [ss_item_sk#32 ASC NULLS FIRST, d_date#31 ASC NULLS FIRST] + +(46) CometColumnarToRow [codegen id : 7] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] + +(47) Window +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [row_number() windowspecdefinition(ss_item_sk#32, d_date#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [ss_item_sk#32], [d_date#31 ASC NULLS FIRST] + +(48) Project [codegen id : 8] +Output [3]: [item_sk#28 AS item_sk#34, sumss#29 AS sumss#35, rk#33] +Input [5]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32, rk#33] + +(49) BroadcastExchange +Input [3]: [item_sk#34, sumss#35, rk#33] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] + +(50) BroadcastHashJoin [codegen id : 9] +Left keys [1]: [item_sk#28] +Right keys [1]: [item_sk#34] Join type: Inner -Join condition: (rk#34 >= rk#37) - -(49) Project [codegen id : 22] -Output [4]: [item_sk#32, d_date#28, sumss#33, sumss#39] -Input [7]: [item_sk#32, d_date#28, sumss#33, rk#34, item_sk#38, sumss#39, rk#37] - -(50) HashAggregate [codegen id : 22] -Input [4]: [item_sk#32, d_date#28, sumss#33, sumss#39] -Keys [3]: [item_sk#32, d_date#28, sumss#33] -Functions [1]: [partial_sum(sumss#39)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] - -(51) Exchange -Input [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] -Arguments: hashpartitioning(item_sk#32, d_date#28, sumss#33, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(52) HashAggregate [codegen id : 23] -Input [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] -Keys [3]: [item_sk#32, d_date#28, sumss#33] -Functions [1]: [sum(sumss#39)] -Aggregate Attributes [1]: [sum(sumss#39)#44] -Results [3]: [item_sk#32, d_date#28, sum(sumss#39)#44 AS cume_sales#45] - -(53) Exchange -Input [3]: [item_sk#32, d_date#28, cume_sales#45] -Arguments: hashpartitioning(item_sk#32, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(54) Sort [codegen id : 24] -Input [3]: [item_sk#32, d_date#28, cume_sales#45] -Arguments: [item_sk#32 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 - -(55) SortMergeJoin [codegen id : 25] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#32, d_date#28] -Join type: FullOuter -Join condition: None - -(56) Filter [codegen id : 25] -Input [6]: [item_sk#10, d_date#5, cume_sales#23, item_sk#32, d_date#28, cume_sales#45] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END) - -(57) Project [codegen id : 25] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END AS item_sk#46, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#28 END AS d_date#47, cume_sales#23 AS web_sales#48, cume_sales#45 AS store_sales#49] -Input [6]: [item_sk#10, d_date#5, cume_sales#23, item_sk#32, d_date#28, cume_sales#45] - -(58) Exchange -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: hashpartitioning(item_sk#46, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(59) Sort [codegen id : 26] -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], false, 0 - -(60) Window -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [row_number() windowspecdefinition(item_sk#46, d_date#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#50], [item_sk#46], [d_date#47 ASC NULLS FIRST] - -(61) ReusedExchange [Reuses operator id: 58] -Output [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] - -(62) Sort [codegen id : 52] -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], false, 0 - -(63) Window -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [row_number() windowspecdefinition(item_sk#46, d_date#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#51], [item_sk#46], [d_date#47 ASC NULLS FIRST] - -(64) Project [codegen id : 53] -Output [4]: [item_sk#46 AS item_sk#52, web_sales#48 AS web_sales#53, store_sales#49 AS store_sales#54, rk#51] -Input [5]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, rk#51] - -(65) BroadcastExchange -Input [4]: [item_sk#52, web_sales#53, store_sales#54, rk#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -(66) BroadcastHashJoin [codegen id : 54] -Left keys [1]: [item_sk#46] -Right keys [1]: [item_sk#52] +Join condition: (rk#30 >= rk#33) + +(51) Project [codegen id : 9] +Output [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Input [7]: [item_sk#28, d_date#26, sumss#29, rk#30, item_sk#34, sumss#35, rk#33] + +(52) HashAggregate [codegen id : 9] +Input [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#36, isEmpty#37] +Results [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] + +(53) CometColumnarExchange +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Arguments: hashpartitioning(item_sk#28, d_date#26, sumss#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(54) CometColumnarToRow [codegen id : 10] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] + +(55) HashAggregate [codegen id : 10] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#40] +Results [3]: [item_sk#28, d_date#26, sum(sumss#35)#40 AS cume_sales#41] + +(56) CometColumnarExchange +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: hashpartitioning(item_sk#28, d_date#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(57) CometSort +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#28, d_date#26, cume_sales#41], [item_sk#28 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#21] +Right output [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#8, d_date#5], [item_sk#28, d_date#26], FullOuter + +(59) CometFilter +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END) + +(60) CometProject +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END AS item_sk#42, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#26 END AS d_date#43, cume_sales#21 AS web_sales#44, cume_sales#41 AS store_sales#45] + +(61) CometExchange +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: hashpartitioning(item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(62) CometSort +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] + +(63) CometColumnarToRow [codegen id : 11] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(64) Window +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#46], [item_sk#42], [d_date#43 ASC NULLS FIRST] + +(65) ReusedExchange [Reuses operator id: 61] +Output [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(66) CometSort +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 22] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(68) Window +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#47], [item_sk#42], [d_date#43 ASC NULLS FIRST] + +(69) Project [codegen id : 23] +Output [4]: [item_sk#42 AS item_sk#48, web_sales#44 AS web_sales#49, store_sales#45 AS store_sales#50, rk#47] +Input [5]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#47] + +(70) BroadcastExchange +Input [4]: [item_sk#48, web_sales#49, store_sales#50, rk#47] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(71) BroadcastHashJoin [codegen id : 24] +Left keys [1]: [item_sk#42] +Right keys [1]: [item_sk#48] Join type: Inner -Join condition: (rk#50 >= rk#51) - -(67) Project [codegen id : 54] -Output [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Input [9]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, rk#50, item_sk#52, web_sales#53, store_sales#54, rk#51] - -(68) HashAggregate [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [partial_max(web_sales#53), partial_max(store_sales#54)] -Aggregate Attributes [2]: [max#55, max#56] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#57, max#58] - -(69) HashAggregate [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#57, max#58] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [max(web_sales#53), max(store_sales#54)] -Aggregate Attributes [2]: [max(web_sales#53)#59, max(store_sales#54)#60] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max(web_sales#53)#59 AS web_cumulative#61, max(store_sales#54)#60 AS store_cumulative#62] - -(70) Filter [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] -Condition : ((isnotnull(web_cumulative#61) AND isnotnull(store_cumulative#62)) AND (web_cumulative#61 > store_cumulative#62)) - -(71) TakeOrderedAndProject -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] -Arguments: 100, [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] +Join condition: (rk#46 >= rk#47) + +(72) Project [codegen id : 24] +Output [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Input [9]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#46, item_sk#48, web_sales#49, store_sales#50, rk#47] + +(73) HashAggregate [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [partial_max(web_sales#49), partial_max(store_sales#50)] +Aggregate Attributes [2]: [max#51, max#52] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] + +(74) HashAggregate [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [max(web_sales#49), max(store_sales#50)] +Aggregate Attributes [2]: [max(web_sales#49)#55, max(store_sales#50)#56] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max(web_sales#49)#55 AS web_cumulative#57, max(store_sales#50)#56 AS store_cumulative#58] + +(75) Filter [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Condition : ((isnotnull(web_cumulative#57) AND isnotnull(store_cumulative#58)) AND (web_cumulative#57 > store_cumulative#58)) + +(76) TakeOrderedAndProject +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Arguments: 100, [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/extended.txt new file mode 100644 index 0000000000..e32bdbf198 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/extended.txt @@ -0,0 +1,188 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- HashAggregate + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- CometColumnarToRow + : : +- CometColumnarExchange + : : +- HashAggregate + : : +- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : : +- CometColumnarToRow + : : : +- CometSort + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometSortMergeJoin + :- CometSort + : +- CometColumnarExchange + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometColumnarExchange + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- Project + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_sales + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 126 out of 172 eligible operators (73%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/simplified.txt index 6256c0684c..ee1947e250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (54) + WholeStageCodegen (24) Filter [web_cumulative,store_cumulative] HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] @@ -7,115 +7,101 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store BroadcastHashJoin [item_sk,item_sk,rk,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (26) - Sort [item_sk,d_date] + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (25) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - Filter [item_sk,item_sk] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (11) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumws] #3 - WholeStageCodegen (10) - HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumws,sumws] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (4) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #5 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - Project [ws_item_sk,ws_sales_price,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Project [item_sk,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (8) - Sort [ws_item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - InputAdapter - WholeStageCodegen (24) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #8 - WholeStageCodegen (23) - HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumss] #9 - WholeStageCodegen (22) - HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumss,sumss] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (16) - Sort [ss_item_sk,d_date] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (15) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] - InputAdapter - Exchange [ss_item_sk,d_date] #11 - WholeStageCodegen (14) - HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_sales_price,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (21) - Project [item_sk,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (20) - Sort [ss_item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (5) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumws] #3 + WholeStageCodegen (4) + HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumws,sumws] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + CometExchange [ws_item_sk] #4 + CometHashAggregate [sum] [item_sk,d_date,sumws,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #5 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + Project [item_sk,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #8 + WholeStageCodegen (10) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumss] #9 + WholeStageCodegen (9) + HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumss,sumss] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sum] [item_sk,d_date,sumss,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #11 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #6 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (8) + Project [item_sk,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 InputAdapter BroadcastExchange #13 - WholeStageCodegen (53) + WholeStageCodegen (23) Project [item_sk,web_sales,store_sales,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (52) - Sort [item_sk,d_date] + WholeStageCodegen (22) + CometColumnarToRow InputAdapter - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 + CometSort [item_sk,d_date,web_sales,store_sales] + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/explain.txt index 1e582841be..0cd88caaff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.call_center (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(4) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) -(6) Filter [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] +(14) CometNativeScan parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_call_center_sk#4] -Right keys [1]: [cc_call_center_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] -(21) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] -(27) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] -(30) Filter [codegen id : 22] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] + +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] -(32) ReusedExchange [Reuses operator id: 23] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(33) HashAggregate [codegen id : 12] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#16] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(36) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#16 AS sum_sales#28, rn#27] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16, rn#27] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(38) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] + +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(41) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#34], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#16 AS sum_sales#35, rn#34] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16, rn#34] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=8] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#34 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] +(47) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#9 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/extended.txt new file mode 100644 index 0000000000..357307b654 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/extended.txt @@ -0,0 +1,90 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- BroadcastHashJoin + :- Project + : +- BroadcastHashJoin + : :- Project + : : +- Filter + : : +- Window + : : +- Filter + : : +- Window + : : +- CometColumnarToRow + : : +- CometSort + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.call_center + : +- BroadcastExchange + : +- Project + : +- Window + : +- CometColumnarToRow + : +- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.call_center + +- BroadcastExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.call_center + +Comet accelerated 69 out of 85 eligible operators (81%). Final plan contains 3 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/simplified.txt index 89e676545d..9656904abd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,cc_name] #1 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometNativeScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,cc_name] #7 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/explain.txt index c0cca65544..7347412d3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/explain.txt @@ -1,533 +1,481 @@ == Physical Plan == -TakeOrderedAndProject (91) -+- * HashAggregate (90) - +- Exchange (89) - +- * HashAggregate (88) - +- Union (87) - :- * HashAggregate (76) - : +- Exchange (75) - : +- * HashAggregate (74) - : +- Union (73) - : :- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- Union (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Project (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) - : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : +- BroadcastExchange (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.store (17) - : :- * HashAggregate (46) - : : +- Exchange (45) - : : +- * HashAggregate (44) - : : +- * Project (43) - : : +- * BroadcastHashJoin Inner BuildRight (42) - : : :- * Project (37) - : : : +- * BroadcastHashJoin Inner BuildRight (36) - : : : :- Union (34) - : : : : :- * Project (29) - : : : : : +- * Filter (28) - : : : : : +- * ColumnarToRow (27) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (26) - : : : : +- * Project (33) - : : : : +- * Filter (32) - : : : : +- * ColumnarToRow (31) - : : : : +- Scan parquet spark_catalog.default.catalog_returns (30) - : : : +- ReusedExchange (35) - : : +- BroadcastExchange (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.catalog_page (38) - : +- * HashAggregate (72) - : +- Exchange (71) - : +- * HashAggregate (70) - : +- * Project (69) - : +- * BroadcastHashJoin Inner BuildRight (68) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- Union (60) - : : : :- * Project (50) - : : : : +- * Filter (49) - : : : : +- * ColumnarToRow (48) - : : : : +- Scan parquet spark_catalog.default.web_sales (47) - : : : +- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildLeft (58) - : : : :- BroadcastExchange (53) - : : : : +- * ColumnarToRow (52) - : : : : +- Scan parquet spark_catalog.default.web_returns (51) - : : : +- * Project (57) - : : : +- * Filter (56) - : : : +- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (61) - : +- BroadcastExchange (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.web_site (64) - :- * HashAggregate (81) - : +- Exchange (80) - : +- * HashAggregate (79) - : +- * HashAggregate (78) - : +- ReusedExchange (77) - +- * HashAggregate (86) - +- Exchange (85) - +- * HashAggregate (84) - +- * HashAggregate (83) - +- ReusedExchange (82) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (84) ++- CometTakeOrderedAndProject (83) + +- CometHashAggregate (82) + +- CometExchange (81) + +- CometHashAggregate (80) + +- CometUnion (79) + :- CometHashAggregate (68) + : +- CometExchange (67) + : +- CometHashAggregate (66) + : +- CometUnion (65) + : :- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometNativeScan parquet spark_catalog.default.store (14) + : :- CometHashAggregate (41) + : : +- CometExchange (40) + : : +- CometHashAggregate (39) + : : +- CometProject (38) + : : +- CometBroadcastHashJoin (37) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometUnion (29) + : : : : :- CometProject (25) + : : : : : +- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (23) + : : : : +- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (26) + : : : +- ReusedExchange (30) + : : +- CometBroadcastExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.catalog_page (33) + : +- CometHashAggregate (64) + : +- CometExchange (63) + : +- CometHashAggregate (62) + : +- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometUnion (52) + : : : :- CometProject (44) + : : : : +- CometFilter (43) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (42) + : : : +- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastExchange (46) + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (45) + : : : +- CometProject (49) + : : : +- CometFilter (48) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (59) + : +- CometProject (58) + : +- CometFilter (57) + : +- CometNativeScan parquet spark_catalog.default.web_site (56) + :- CometHashAggregate (73) + : +- CometExchange (72) + : +- CometHashAggregate (71) + : +- CometHashAggregate (70) + : +- ReusedExchange (69) + +- CometHashAggregate (78) + +- CometExchange (77) + +- CometHashAggregate (76) + +- CometHashAggregate (75) + +- ReusedExchange (74) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] +(3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Condition : isnotnull(sr_store_sk#11) -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] +(6) CometProject Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] -(9) Union +(7) CometUnion +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(10) Scan parquet spark_catalog.default.date_dim +(8) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] - -(12) Filter [codegen id : 3] +(9) CometFilter Input [2]: [d_date_sk#21, d_date#22] Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) -(13) Project [codegen id : 3] -Output [1]: [d_date_sk#21] +(10) CometProject Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#21] -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [date_sk#6] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight -(16) Project [codegen id : 5] -Output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +(13) CometProject Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(17) Scan parquet spark_catalog.default.store +(14) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] - -(19) Filter [codegen id : 4] +(15) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(20) BroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] + +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [store_sk#5] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(22) Project [codegen id : 5] -Output [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(23) HashAggregate [codegen id : 5] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -(24) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 6] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] +(22) CometHashAggregate +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [store channel AS channel#37, concat(store, s_store_id#24) AS id#38, MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#40, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#41] -(26) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(23) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(24) CometFilter +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : isnotnull(cs_catalog_page_sk#30) -(28) Filter [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : isnotnull(cs_catalog_page_sk#42) +(25) CometProject +Input [4]: [cs_catalog_page_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39], [cs_catalog_page_sk#30 AS page_sk#34, cs_sold_date_sk#33 AS date_sk#35, cs_ext_sales_price#31 AS sales_price#36, cs_net_profit#32 AS profit#37, 0.00 AS return_amt#38, 0.00 AS net_loss#39] -(29) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#46, cs_sold_date_sk#45 AS date_sk#47, cs_ext_sales_price#43 AS sales_price#48, cs_net_profit#44 AS profit#49, 0.00 AS return_amt#50, 0.00 AS net_loss#51] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(30) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(26) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#55)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] - -(32) Filter [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Condition : isnotnull(cr_catalog_page_sk#52) +(27) CometFilter +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Condition : isnotnull(cr_catalog_page_sk#40) -(33) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#52 AS page_sk#56, cr_returned_date_sk#55 AS date_sk#57, 0.00 AS sales_price#58, 0.00 AS profit#59, cr_return_amount#53 AS return_amt#60, cr_net_loss#54 AS net_loss#61] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(28) CometProject +Input [4]: [cr_catalog_page_sk#40, cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Arguments: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49], [cr_catalog_page_sk#40 AS page_sk#44, cr_returned_date_sk#43 AS date_sk#45, 0.00 AS sales_price#46, 0.00 AS profit#47, cr_return_amount#41 AS return_amt#48, cr_net_loss#42 AS net_loss#49] -(34) Union +(29) CometUnion +Child 0 Input [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Child 1 Input [6]: [page_sk#44, date_sk#45, sales_price#46, profit#47, return_amt#48, net_loss#49] -(35) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#62] +(30) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#50] -(36) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [date_sk#47] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None +(31) CometBroadcastHashJoin +Left output [6]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [1]: [d_date_sk#50] +Arguments: [date_sk#35], [d_date_sk#50], Inner, BuildRight -(37) Project [codegen id : 11] -Output [5]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51] -Input [7]: [page_sk#46, date_sk#47, sales_price#48, profit#49, return_amt#50, net_loss#51, d_date_sk#62] +(32) CometProject +Input [7]: [page_sk#34, date_sk#35, sales_price#36, profit#37, return_amt#38, net_loss#39, d_date_sk#50] +Arguments: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39], [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] -(38) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] +(33) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(40) Filter [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Condition : isnotnull(cp_catalog_page_sk#63) - -(41) BroadcastExchange -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(42) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [page_sk#46] -Right keys [1]: [cp_catalog_page_sk#63] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 11] -Output [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Input [7]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(44) HashAggregate [codegen id : 11] -Input [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#48)), partial_sum(UnscaledValue(return_amt#50)), partial_sum(UnscaledValue(profit#49)), partial_sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum#65, sum#66, sum#67, sum#68] -Results [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] - -(45) Exchange -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(46) HashAggregate [codegen id : 12] -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#48)), sum(UnscaledValue(return_amt#50)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#48))#73, sum(UnscaledValue(return_amt#50))#74, sum(UnscaledValue(profit#49))#75, sum(UnscaledValue(net_loss#51))#76] -Results [5]: [catalog channel AS channel#77, concat(catalog_page, cp_catalog_page_id#64) AS id#78, MakeDecimal(sum(UnscaledValue(sales_price#48))#73,17,2) AS sales#79, MakeDecimal(sum(UnscaledValue(return_amt#50))#74,17,2) AS returns#80, (MakeDecimal(sum(UnscaledValue(profit#49))#75,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#51))#76,17,2)) AS profit#81] - -(47) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(34) CometFilter +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Condition : isnotnull(cp_catalog_page_sk#51) + +(35) CometProject +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53], [cp_catalog_page_sk#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#52, 16, true, false, true) AS cp_catalog_page_id#53] + +(36) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [cp_catalog_page_sk#51, cp_catalog_page_id#53] + +(37) CometBroadcastHashJoin +Left output [5]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39] +Right output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [page_sk#34], [cp_catalog_page_sk#51], Inner, BuildRight + +(38) CometProject +Input [7]: [page_sk#34, sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_sk#51, cp_catalog_page_id#53] +Arguments: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53], [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] + +(39) CometHashAggregate +Input [5]: [sales_price#36, profit#37, return_amt#38, net_loss#39, cp_catalog_page_id#53] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [partial_sum(UnscaledValue(sales_price#36)), partial_sum(UnscaledValue(return_amt#38)), partial_sum(UnscaledValue(profit#37)), partial_sum(UnscaledValue(net_loss#39))] + +(40) CometExchange +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(41) CometHashAggregate +Input [5]: [cp_catalog_page_id#53, sum#54, sum#55, sum#56, sum#57] +Keys [1]: [cp_catalog_page_id#53] +Functions [4]: [sum(UnscaledValue(sales_price#36)), sum(UnscaledValue(return_amt#38)), sum(UnscaledValue(profit#37)), sum(UnscaledValue(net_loss#39))] + +(42) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#85)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(49) Filter [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Condition : isnotnull(ws_web_site_sk#82) +(43) CometFilter +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_web_site_sk#58) -(50) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#82 AS wsr_web_site_sk#86, ws_sold_date_sk#85 AS date_sk#87, ws_ext_sales_price#83 AS sales_price#88, ws_net_profit#84 AS profit#89, 0.00 AS return_amt#90, 0.00 AS net_loss#91] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(44) CometProject +Input [4]: [ws_web_site_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Arguments: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67], [ws_web_site_sk#58 AS wsr_web_site_sk#62, ws_sold_date_sk#61 AS date_sk#63, ws_ext_sales_price#59 AS sales_price#64, ws_net_profit#60 AS profit#65, 0.00 AS return_amt#66, 0.00 AS net_loss#67] -(51) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] +(45) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#96)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#72)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] - -(53) BroadcastExchange -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=6] +(46) CometBroadcastExchange +Input [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Arguments: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(47) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(55) ColumnarToRow -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(48) CometFilter +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Condition : ((isnotnull(ws_item_sk#73) AND isnotnull(ws_order_number#75)) AND isnotnull(ws_web_site_sk#74)) -(56) Filter -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Condition : ((isnotnull(ws_item_sk#97) AND isnotnull(ws_order_number#99)) AND isnotnull(ws_web_site_sk#98)) +(49) CometProject +Input [4]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75, ws_sold_date_sk#76] +Arguments: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75], [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] -(57) Project -Output [3]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(50) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72] +Right output [3]: [ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wr_item_sk#68, wr_order_number#69], [ws_item_sk#73, ws_order_number#75], Inner, BuildLeft -(58) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#92, wr_order_number#93] -Right keys [2]: [ws_item_sk#97, ws_order_number#99] -Join type: Inner -Join condition: None +(51) CometProject +Input [8]: [wr_item_sk#68, wr_order_number#69, wr_return_amt#70, wr_net_loss#71, wr_returned_date_sk#72, ws_item_sk#73, ws_web_site_sk#74, ws_order_number#75] +Arguments: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82], [ws_web_site_sk#74 AS wsr_web_site_sk#77, wr_returned_date_sk#72 AS date_sk#78, 0.00 AS sales_price#79, 0.00 AS profit#80, wr_return_amt#70 AS return_amt#81, wr_net_loss#71 AS net_loss#82] -(59) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#98 AS wsr_web_site_sk#101, wr_returned_date_sk#96 AS date_sk#102, 0.00 AS sales_price#103, 0.00 AS profit#104, wr_return_amt#94 AS return_amt#105, wr_net_loss#95 AS net_loss#106] -Input [8]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96, ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] +(52) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Child 1 Input [6]: [wsr_web_site_sk#77, date_sk#78, sales_price#79, profit#80, return_amt#81, net_loss#82] -(60) Union +(53) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#83] -(61) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#107] +(54) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [1]: [d_date_sk#83] +Arguments: [date_sk#63], [d_date_sk#83], Inner, BuildRight -(62) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [date_sk#87] -Right keys [1]: [d_date_sk#107] -Join type: Inner -Join condition: None +(55) CometProject +Input [7]: [wsr_web_site_sk#62, date_sk#63, sales_price#64, profit#65, return_amt#66, net_loss#67, d_date_sk#83] +Arguments: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67], [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] -(63) Project [codegen id : 18] -Output [5]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91] -Input [7]: [wsr_web_site_sk#86, date_sk#87, sales_price#88, profit#89, return_amt#90, net_loss#91, d_date_sk#107] - -(64) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#108, web_site_id#109] +(56) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#84, web_site_id#85] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] - -(66) Filter [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] -Condition : isnotnull(web_site_sk#108) - -(67) BroadcastExchange -Input [2]: [web_site_sk#108, web_site_id#109] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(68) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [wsr_web_site_sk#86] -Right keys [1]: [web_site_sk#108] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 18] -Output [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Input [7]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_sk#108, web_site_id#109] - -(70) HashAggregate [codegen id : 18] -Input [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Keys [1]: [web_site_id#109] -Functions [4]: [partial_sum(UnscaledValue(sales_price#88)), partial_sum(UnscaledValue(return_amt#90)), partial_sum(UnscaledValue(profit#89)), partial_sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum#110, sum#111, sum#112, sum#113] -Results [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] - -(71) Exchange -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Arguments: hashpartitioning(web_site_id#109, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(72) HashAggregate [codegen id : 19] -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Keys [1]: [web_site_id#109] -Functions [4]: [sum(UnscaledValue(sales_price#88)), sum(UnscaledValue(return_amt#90)), sum(UnscaledValue(profit#89)), sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#88))#118, sum(UnscaledValue(return_amt#90))#119, sum(UnscaledValue(profit#89))#120, sum(UnscaledValue(net_loss#91))#121] -Results [5]: [web channel AS channel#122, concat(web_site, web_site_id#109) AS id#123, MakeDecimal(sum(UnscaledValue(sales_price#88))#118,17,2) AS sales#124, MakeDecimal(sum(UnscaledValue(return_amt#90))#119,17,2) AS returns#125, (MakeDecimal(sum(UnscaledValue(profit#89))#120,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#91))#121,17,2)) AS profit#126] - -(73) Union - -(74) HashAggregate [codegen id : 20] -Input [5]: [channel#37, id#38, sales#39, returns#40, profit#41] -Keys [2]: [channel#37, id#38] -Functions [3]: [partial_sum(sales#39), partial_sum(returns#40), partial_sum(profit#41)] -Aggregate Attributes [6]: [sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Results [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(75) Exchange -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Arguments: hashpartitioning(channel#37, id#38, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(76) HashAggregate [codegen id : 21] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [5]: [channel#37, id#38, cast(sum(sales#39)#139 as decimal(37,2)) AS sales#142, cast(sum(returns#40)#140 as decimal(37,2)) AS returns#143, cast(sum(profit#41)#141 as decimal(38,2)) AS profit#144] - -(77) ReusedExchange [Reuses operator id: 75] -Output [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(78) HashAggregate [codegen id : 42] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [4]: [channel#37, sum(sales#39)#139 AS sales#145, sum(returns#40)#140 AS returns#146, sum(profit#41)#141 AS profit#147] - -(79) HashAggregate [codegen id : 42] -Input [4]: [channel#37, sales#145, returns#146, profit#147] -Keys [1]: [channel#37] -Functions [3]: [partial_sum(sales#145), partial_sum(returns#146), partial_sum(profit#147)] -Aggregate Attributes [6]: [sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] -Results [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] - -(80) Exchange -Input [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] -Arguments: hashpartitioning(channel#37, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(81) HashAggregate [codegen id : 43] -Input [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] -Keys [1]: [channel#37] -Functions [3]: [sum(sales#145), sum(returns#146), sum(profit#147)] -Aggregate Attributes [3]: [sum(sales#145)#160, sum(returns#146)#161, sum(profit#147)#162] -Results [5]: [channel#37, null AS id#163, sum(sales#145)#160 AS sum(sales)#164, sum(returns#146)#161 AS sum(returns)#165, sum(profit#147)#162 AS sum(profit)#166] - -(82) ReusedExchange [Reuses operator id: 75] -Output [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(83) HashAggregate [codegen id : 64] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [3]: [sum(sales#39)#139 AS sales#145, sum(returns#40)#140 AS returns#146, sum(profit#41)#141 AS profit#147] - -(84) HashAggregate [codegen id : 64] -Input [3]: [sales#145, returns#146, profit#147] +(57) CometFilter +Input [2]: [web_site_sk#84, web_site_id#85] +Condition : isnotnull(web_site_sk#84) + +(58) CometProject +Input [2]: [web_site_sk#84, web_site_id#85] +Arguments: [web_site_sk#84, web_site_id#86], [web_site_sk#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#85, 16, true, false, true) AS web_site_id#86] + +(59) CometBroadcastExchange +Input [2]: [web_site_sk#84, web_site_id#86] +Arguments: [web_site_sk#84, web_site_id#86] + +(60) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67] +Right output [2]: [web_site_sk#84, web_site_id#86] +Arguments: [wsr_web_site_sk#62], [web_site_sk#84], Inner, BuildRight + +(61) CometProject +Input [7]: [wsr_web_site_sk#62, sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_sk#84, web_site_id#86] +Arguments: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86], [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] + +(62) CometHashAggregate +Input [5]: [sales_price#64, profit#65, return_amt#66, net_loss#67, web_site_id#86] +Keys [1]: [web_site_id#86] +Functions [4]: [partial_sum(UnscaledValue(sales_price#64)), partial_sum(UnscaledValue(return_amt#66)), partial_sum(UnscaledValue(profit#65)), partial_sum(UnscaledValue(net_loss#67))] + +(63) CometExchange +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Arguments: hashpartitioning(web_site_id#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(64) CometHashAggregate +Input [5]: [web_site_id#86, sum#87, sum#88, sum#89, sum#90] +Keys [1]: [web_site_id#86] +Functions [4]: [sum(UnscaledValue(sales_price#64)), sum(UnscaledValue(return_amt#66)), sum(UnscaledValue(profit#65)), sum(UnscaledValue(net_loss#67))] + +(65) CometUnion +Child 0 Input [5]: [channel#91, id#92, sales#93, returns#94, profit#95] +Child 1 Input [5]: [channel#96, id#97, sales#98, returns#99, profit#100] +Child 2 Input [5]: [channel#101, id#102, sales#103, returns#104, profit#105] + +(66) CometHashAggregate +Input [5]: [channel#91, id#92, sales#93, returns#94, profit#95] +Keys [2]: [channel#91, id#92] +Functions [3]: [partial_sum(sales#93), partial_sum(returns#94), partial_sum(profit#95)] + +(67) CometExchange +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#91, id#92, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(68) CometHashAggregate +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [2]: [channel#91, id#92] +Functions [3]: [sum(sales#93), sum(returns#94), sum(profit#95)] + +(69) ReusedExchange [Reuses operator id: 67] +Output [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(70) CometHashAggregate +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [2]: [channel#91, id#92] +Functions [3]: [sum(sales#93), sum(returns#94), sum(profit#95)] + +(71) CometHashAggregate +Input [4]: [channel#91, sales#112, returns#113, profit#114] +Keys [1]: [channel#91] +Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] + +(72) CometExchange +Input [7]: [channel#91, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] +Arguments: hashpartitioning(channel#91, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(73) CometHashAggregate +Input [7]: [channel#91, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] +Keys [1]: [channel#91] +Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] + +(74) ReusedExchange [Reuses operator id: 67] +Output [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(75) CometHashAggregate +Input [8]: [channel#91, id#92, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [2]: [channel#91, id#92] +Functions [3]: [sum(sales#93), sum(returns#94), sum(profit#95)] + +(76) CometHashAggregate +Input [3]: [sales#112, returns#113, profit#114] Keys: [] -Functions [3]: [partial_sum(sales#145), partial_sum(returns#146), partial_sum(profit#147)] -Aggregate Attributes [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -Results [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] +Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -(85) Exchange -Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +(77) CometExchange +Input [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(86) HashAggregate [codegen id : 65] -Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] +(78) CometHashAggregate +Input [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] Keys: [] -Functions [3]: [sum(sales#145), sum(returns#146), sum(profit#147)] -Aggregate Attributes [3]: [sum(sales#145)#179, sum(returns#146)#180, sum(profit#147)#181] -Results [5]: [null AS channel#182, null AS id#183, sum(sales#145)#179 AS sum(sales)#184, sum(returns#146)#180 AS sum(returns)#185, sum(profit#147)#181 AS sum(profit)#186] +Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -(87) Union +(79) CometUnion +Child 0 Input [5]: [channel#91, id#92, sales#127, returns#128, profit#129] +Child 1 Input [5]: [channel#91, id#130, sum(sales)#131, sum(returns)#132, sum(profit)#133] +Child 2 Input [5]: [channel#134, id#135, sum(sales)#136, sum(returns)#137, sum(profit)#138] -(88) HashAggregate [codegen id : 66] -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Keys [5]: [channel#37, id#38, sales#142, returns#143, profit#144] +(80) CometHashAggregate +Input [5]: [channel#91, id#92, sales#127, returns#128, profit#129] +Keys [5]: [channel#91, id#92, sales#127, returns#128, profit#129] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -(89) Exchange -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Arguments: hashpartitioning(channel#37, id#38, sales#142, returns#143, profit#144, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(81) CometExchange +Input [5]: [channel#91, id#92, sales#127, returns#128, profit#129] +Arguments: hashpartitioning(channel#91, id#92, sales#127, returns#128, profit#129, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(90) HashAggregate [codegen id : 67] -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Keys [5]: [channel#37, id#38, sales#142, returns#143, profit#144] +(82) CometHashAggregate +Input [5]: [channel#91, id#92, sales#127, returns#128, profit#129] +Keys [5]: [channel#91, id#92, sales#127, returns#128, profit#129] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -(91) TakeOrderedAndProject -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Arguments: 100, [channel#37 ASC NULLS FIRST, id#38 ASC NULLS FIRST], [channel#37, id#38, sales#142, returns#143, profit#144] +(83) CometTakeOrderedAndProject +Input [5]: [channel#91, id#92, sales#127, returns#128, profit#129] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#91 ASC NULLS FIRST,id#92 ASC NULLS FIRST], output=[channel#91,id#92,sales#127,returns#128,profit#129]), [channel#91, id#92, sales#127, returns#128, profit#129], 100, 0, [channel#91 ASC NULLS FIRST, id#92 ASC NULLS FIRST], [channel#91, id#92, sales#127, returns#128, profit#129] + +(84) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#91, id#92, sales#127, returns#128, profit#129] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/extended.txt new file mode 100644 index 0000000000..3b96b1117a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/extended.txt @@ -0,0 +1,236 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastExchange + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometUnion + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometBroadcastExchange + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_site + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometUnion + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_page + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometUnion + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometBroadcastExchange + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_site + +Comet accelerated 233 out of 233 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/simplified.txt index a67882dfb4..73ebe96769 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/simplified.txt @@ -1,148 +1,86 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (67) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (66) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (21) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (20) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_id] #3 - WholeStageCodegen (5) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,s_store_id] - BroadcastHashJoin [store_sk,s_store_sk] - Project [store_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - InputAdapter - Exchange [cp_catalog_page_id] #6 - WholeStageCodegen (11) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - BroadcastHashJoin [page_sk,cp_catalog_page_sk] - Project [page_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #8 - WholeStageCodegen (18) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,web_site_id] - BroadcastHashJoin [wsr_web_site_sk,web_site_sk] - Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (43) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #11 - WholeStageCodegen (42) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (65) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #12 - WholeStageCodegen (64) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometExchange [channel,id,sales,returns,profit] #1 + CometHashAggregate [channel,id,sales,returns,profit] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id] #2 + CometHashAggregate [sales,returns,profit] [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,sum,sum,sum] [channel,id,sales,returns,profit,s_store_id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss))] + CometExchange [s_store_id] #3 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometHashAggregate [sum,sum,sum,sum] [channel,id,sales,returns,profit,cp_catalog_page_id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss))] + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometHashAggregate [sum,sum,sum,sum] [channel,id,sales,returns,profit,web_site_id,sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss))] + CometExchange [web_site_id] #8 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sum(sales),sum(returns),sum(profit),sum(sales),sum(returns),sum(profit)] + CometExchange [channel] #11 + CometHashAggregate [sales,returns,profit] [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sum(sales),sum(returns),sum(profit),sum(sales),sum(returns),sum(profit)] + CometExchange #12 + CometHashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/explain.txt index eaed1f54a0..74bd8c52cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/explain.txt @@ -1,302 +1,274 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Project (38) - +- * BroadcastHashJoin Inner BuildRight (37) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer_address (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store_sales (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Filter (32) - +- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.item (26) - - -(1) Scan parquet spark_catalog.default.customer_address +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometNativeScan parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometNativeScan parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometNativeScan parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometNativeScan parquet spark_catalog.default.item (22) + + +(1) CometNativeScan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) -(3) Filter [codegen id : 7] +(3) CometProject Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] +(4) CometNativeScan parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) -(7) BroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ca_address_sk#1] -Right keys [1]: [c_current_addr_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight -(9) Project [codegen id : 7] -Output [2]: [ca_state#2, c_customer_sk#3] -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(10) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(9) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) -(12) Filter [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] -(13) BroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: Inner -Join condition: None +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(15) Project [codegen id : 7] -Output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +(14) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(18) Filter [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#8] -Input [2]: [d_date_sk#8, d_month_seq#9] +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(20) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(22) Project [codegen id : 7] -Output [2]: [ca_state#2, ss_item_sk#5] -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(23) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(20) CometNativeScan parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) -(25) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(26) Scan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +(22) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(23) CometFilter +Input [2]: [i_current_price#14, i_category#15] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)) -(28) Filter [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) +(24) CometProject +Input [2]: [i_current_price#14, i_category#15] +Arguments: [i_current_price#14, i_category#16], [i_current_price#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) AS i_category#16] -(29) HashAggregate [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(25) CometHashAggregate +Input [2]: [i_current_price#14, i_category#16] Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [i_category#16, sum#19, count#20] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#14))] -(30) Exchange -Input [3]: [i_category#16, sum#19, count#20] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(26) CometExchange +Input [3]: [i_category#16, sum#17, count#18] +Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 5] -Input [3]: [i_category#16, sum#19, count#20] +(27) CometHashAggregate +Input [3]: [i_category#16, sum#17, count#18] Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#15))#21] -Results [2]: [cast((avg(UnscaledValue(i_current_price#15))#21 / 100.0) as decimal(11,6)) AS avg(i_current_price)#22, i_category#16] - -(32) Filter [codegen id : 5] -Input [2]: [avg(i_current_price)#22, i_category#16] -Condition : isnotnull(avg(i_current_price)#22) - -(33) BroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#16] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_category#14] -Right keys [1]: [i_category#16] -Join type: Inner -Join condition: (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)) - -(35) Project [codegen id : 6] -Output [1]: [i_item_sk#12] -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#22, i_category#16] - -(36) BroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#5] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [1]: [ca_state#2] -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] - -(39) HashAggregate [codegen id : 7] -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] +Functions [1]: [avg(UnscaledValue(i_current_price#14))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#19, i_category#16] +Condition : isnotnull(avg(i_current_price)#19) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [avg(i_current_price)#19, i_category#16] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#16], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight + +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#19, i_category#16] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] + +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [2]: [ca_state#2, count#24] -(40) Exchange -Input [2]: [ca_state#2, count#24] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(36) CometExchange +Input [2]: [ca_state#3, count#20] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(41) HashAggregate [codegen id : 8] -Input [2]: [ca_state#2, count#24] -Keys [1]: [ca_state#2] +(37) CometHashAggregate +Input [2]: [ca_state#3, count#20] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [3]: [ca_state#2 AS state#26, count(1)#25 AS cnt#27, ca_state#2] -(42) Filter [codegen id : 8] -Input [3]: [state#26, cnt#27, ca_state#2] -Condition : (cnt#27 >= 10) +(38) CometFilter +Input [3]: [state#21, cnt#22, ca_state#3] +Condition : (cnt#22 >= 10) + +(39) CometTakeOrderedAndProject +Input [3]: [state#21, cnt#22, ca_state#3] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, 0, [cnt#22 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#21, cnt#22] -(43) TakeOrderedAndProject -Input [3]: [state#26, cnt#27, ca_state#2] -Arguments: 100, [cnt#27 ASC NULLS FIRST, ca_state#2 ASC NULLS FIRST], [state#26, cnt#27] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#21, cnt#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) - +- Scan parquet spark_catalog.default.date_dim (44) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometNativeScan parquet spark_catalog.default.date_dim (41) -(44) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#28, d_year#29, d_moy#30] +(41) CometNativeScan parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#10, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] +(42) CometFilter +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Condition : (((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2000)) AND (d_moy#24 = 1)) -(46) Filter [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] -Condition : (((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2000)) AND (d_moy#30 = 1)) +(43) CometProject +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Arguments: [d_month_seq#10], [d_month_seq#10] -(47) Project [codegen id : 1] -Output [1]: [d_month_seq#28] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(48) HashAggregate [codegen id : 1] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(44) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] -(49) Exchange -Input [1]: [d_month_seq#28] -Arguments: hashpartitioning(d_month_seq#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(45) CometExchange +Input [1]: [d_month_seq#10] +Arguments: hashpartitioning(d_month_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(50) HashAggregate [codegen id : 2] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(46) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] + +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/extended.txt new file mode 100644 index 0000000000..f6467c6fd9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/extended.txt @@ -0,0 +1,50 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : : +- Subquery + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 45 out of 46 eligible operators (97%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/simplified.txt index 9a2eddad66..6f55ccc7cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/simplified.txt @@ -1,76 +1,52 @@ -TakeOrderedAndProject [cnt,ca_state,state] - WholeStageCodegen (8) - Filter [cnt] - HashAggregate [ca_state,count] [count(1),state,cnt,count] - InputAdapter - Exchange [ca_state] #1 - WholeStageCodegen (7) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ca_state,ss_item_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ca_state,ss_item_sk,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Project [ca_state,c_customer_sk] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [state,cnt,ca_state] + CometFilter [state,cnt,ca_state] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] + CometExchange [ca_state] #1 + CometHashAggregate [ca_state,count] + CometProject [ca_state] + CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [d_month_seq] - InputAdapter - Exchange [d_month_seq] #5 - WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] - InputAdapter - Exchange [i_category] #8 - WholeStageCodegen (4) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_current_price,i_category] + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk] #6 + CometProject [i_item_sk] + CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] + CometFilter [i_item_sk,i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometBroadcastExchange [avg(i_current_price),i_category] #7 + CometFilter [avg(i_current_price),i_category] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] + CometExchange [i_category] #8 + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometNativeScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/explain.txt index b557c1012c..3ad67801f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/explain.txt @@ -1,999 +1,909 @@ == Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) - :- * Sort (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- * Project (110) - : +- * BroadcastHashJoin Inner BuildRight (109) - : :- * Project (103) - : : +- * BroadcastHashJoin Inner BuildRight (102) - : : :- * Project (100) - : : : +- * BroadcastHashJoin Inner BuildRight (99) - : : : :- * Project (94) - : : : : +- * BroadcastHashJoin Inner BuildRight (93) - : : : : :- * Project (91) - : : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : : :- * Project (85) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) - : : : : : : :- * Project (82) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : : :- * Project (76) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) - : : : : : : : : :- * Project (70) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) - : : : : : : : : : :- * Project (67) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : : :- * Project (61) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : : :- * Project (52) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (51) - : : : : : : : : : : : : : :- * Project (46) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (45) - : : : : : : : : : : : : : : :- * Project (40) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (39) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- BroadcastExchange (38) - : : : : : : : : : : : : : : : +- * Filter (37) - : : : : : : : : : : : : : : : +- * ColumnarToRow (36) - : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (35) - : : : : : : : : : : : : : : +- BroadcastExchange (44) - : : : : : : : : : : : : : : +- * Filter (43) - : : : : : : : : : : : : : : +- * ColumnarToRow (42) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store (41) - : : : : : : : : : : : : : +- BroadcastExchange (50) - : : : : : : : : : : : : : +- * Filter (49) - : : : : : : : : : : : : : +- * ColumnarToRow (48) - : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.customer (47) - : : : : : : : : : : : : +- BroadcastExchange (56) - : : : : : : : : : : : : +- * Filter (55) - : : : : : : : : : : : : +- * ColumnarToRow (54) - : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (53) - : : : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : : : +- BroadcastExchange (65) - : : : : : : : : : : +- * Filter (64) - : : : : : : : : : : +- * ColumnarToRow (63) - : : : : : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (62) - : : : : : : : : : +- ReusedExchange (68) - : : : : : : : : +- BroadcastExchange (74) - : : : : : : : : +- * Filter (73) - : : : : : : : : +- * ColumnarToRow (72) - : : : : : : : : +- Scan parquet spark_catalog.default.promotion (71) - : : : : : : : +- BroadcastExchange (80) - : : : : : : : +- * Filter (79) - : : : : : : : +- * ColumnarToRow (78) - : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (77) - : : : : : : +- ReusedExchange (83) - : : : : : +- BroadcastExchange (89) - : : : : : +- * Filter (88) - : : : : : +- * ColumnarToRow (87) - : : : : : +- Scan parquet spark_catalog.default.customer_address (86) - : : : : +- ReusedExchange (92) - : : : +- BroadcastExchange (98) - : : : +- * Filter (97) - : : : +- * ColumnarToRow (96) - : : : +- Scan parquet spark_catalog.default.income_band (95) - : : +- ReusedExchange (101) - : +- BroadcastExchange (108) - : +- * Project (107) - : +- * Filter (106) - : +- * ColumnarToRow (105) - : +- Scan parquet spark_catalog.default.item (104) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * Project (123) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (122) - : : : : : : : : : : : : : : : :- * Sort (116) - : : : : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : : : : +- * Sort (121) - : : : : : : : : : : : : : : : +- * Project (120) - : : : : : : : : : : : : : : : +- * Filter (119) - : : : : : : : : : : : : : : : +- * HashAggregate (118) - : : : : : : : : : : : : : : : +- ReusedExchange (117) - : : : : : : : : : : : : : : +- BroadcastExchange (127) - : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : +- * ColumnarToRow (125) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (124) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) + : : : : : : : : : : : : : : :- CometProject (35) + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) + : : : : : : : : : : : : : : : :- CometProject (30) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) + : : : : : : : : : : : : : : : : :- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- CometSort (28) + : : : : : : : : : : : : : : : : +- CometProject (27) + : : : : : : : : : : : : : : : : +- CometFilter (26) + : : : : : : : : : : : : : : : : +- CometHashAggregate (25) + : : : : : : : : : : : : : : : : +- CometExchange (24) + : : : : : : : : : : : : : : : : +- CometHashAggregate (23) + : : : : : : : : : : : : : : : : +- CometProject (22) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) + : : : : : : : : : : : : : : : : :- CometSort (15) + : : : : : : : : : : : : : : : : : +- CometExchange (14) + : : : : : : : : : : : : : : : : : +- CometProject (13) + : : : : : : : : : : : : : : : : : +- CometFilter (12) + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : +- CometSort (20) + : : : : : : : : : : : : : : : : +- CometExchange (19) + : : : : : : : : : : : : : : : : +- CometProject (18) + : : : : : : : : : : : : : : : : +- CometFilter (17) + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) + : : : : : : : : : : : : : : : +- CometFilter (32) + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometNativeScan parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometNativeScan parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(6) ColumnarToRow -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(7) Filter +(5) CometFilter Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) -(8) Project -Output [2]: [sr_item_sk#13, sr_ticket_number#14] +(6) CometProject Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#13, sr_ticket_number#14] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(8) CometProject Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(11) Exchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) Sort [codegen id : 3] +(10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(13) Scan parquet spark_catalog.default.catalog_sales +(11) CometNativeScan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(15) Filter [codegen id : 4] +(12) CometFilter Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +(13) CometProject Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -(17) Exchange +(14) CometExchange Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) Sort [codegen id : 5] +(15) CometSort Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(21) Filter [codegen id : 6] +(17) CometFilter Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(18) CometProject Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(23) Exchange +(19) CometExchange Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(24) Sort [codegen id : 7] +(20) CometSort Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_item_sk#16, cs_order_number#17] -Right keys [2]: [cr_item_sk#20, cr_order_number#21] -Join type: Inner -Join condition: None +(21) CometSortMergeJoin +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner -(26) Project [codegen id : 8] -Output [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(22) CometProject Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(27) HashAggregate [codegen id : 8] +(23) CometHashAggregate Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] Keys [1]: [cs_item_sk#16] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -(28) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(29) HashAggregate [codegen id : 9] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] +(25) CometHashAggregate +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] Keys [1]: [cs_item_sk#16] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] -(30) Filter [codegen id : 9] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) +(26) CometFilter +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(31) Project [codegen id : 9] -Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] +(27) CometProject +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] -(32) Sort [codegen id : 9] +(28) CometSort Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] -(33) SortMergeJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#16] -Join type: Inner -Join condition: None +(29) CometSortMergeJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner -(34) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(30) CometProject Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(35) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#36, d_year#37] +(31) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] +(32) CometFilter +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) -(37) Filter [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) +(33) CometBroadcastExchange +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] -(38) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(34) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None +(35) CometProject +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(40) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] - -(41) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(36) CometNativeScan parquet spark_catalog.default.store +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(37) CometFilter +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(43) Filter [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) +(38) CometProject +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] -(44) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] -(45) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None +(40) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(46) Project [codegen id : 25] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] -(47) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(42) CometNativeScan parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(49) Filter [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) -(50) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(51) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] -Join type: Inner -Join condition: None +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight -(52) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(53) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#47, d_year#48] +(47) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) -(55) Filter [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(56) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight -(57) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] -Join type: Inner -Join condition: None +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] -(58) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#49, d_year#50] +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight -(60) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] -Join type: Inner -Join condition: None +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(61) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] - -(62) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +(55) CometNativeScan parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(63) ColumnarToRow [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) -(64) Filter [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] -(65) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] -(66) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight -(67) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] -(68) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] -Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(70) Project [codegen id : 25] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(71) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] +(64) CometNativeScan parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 17] -Input [1]: [p_promo_sk#55] - -(73) Filter [codegen id : 17] -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) -(74) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] -(75) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight -(76) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(77) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(69) CometNativeScan parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(78) ColumnarToRow [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(79) Filter [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(80) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight -(81) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] -Join type: Inner -Join condition: None +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(82) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(83) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(84) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] -Join type: Inner -Join condition: None +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(85) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] - -(86) Scan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(77) CometNativeScan parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(87) ColumnarToRow [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) -(88) Filter [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) +(79) CometProject +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(89) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(90) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] -Join type: Inner -Join condition: None +(81) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight -(91) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(82) CometProject +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(92) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(93) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] -Join type: Inner -Join condition: None +(84) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight -(94) Project [codegen id : 25] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(85) CometProject +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(95) Scan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +(86) CometNativeScan parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 22] -Input [1]: [ib_income_band_sk#70] - -(97) Filter [codegen id : 22] -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) -(98) BroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] -(99) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] -Join type: Inner -Join condition: None +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight -(100) Project [codegen id : 25] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(101) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#71] +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] -(102) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight -(103) Project [codegen id : 25] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(104) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(94) CometNativeScan parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(105) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(106) Filter [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) -(107) Project [codegen id : 24] -Output [2]: [i_item_sk#72, i_product_name#75] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] -(108) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] -(109) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight -(110) Project [codegen id : 25] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] -(111) HashAggregate [codegen id : 25] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -(112) HashAggregate [codegen id : 25] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] - -(113) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(114) Sort [codegen id : 26] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 - -(115) ReusedExchange [Reuses operator id: 11] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(116) Sort [codegen id : 29] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105 ASC NULLS FIRST], false, 0 - -(117) ReusedExchange [Reuses operator id: 28] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] - -(118) HashAggregate [codegen id : 35] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#34, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#35] - -(119) Filter [codegen id : 35] -Input [3]: [cs_item_sk#116, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(120) Project [codegen id : 35] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#34, refund#35] - -(121) Sort [codegen id : 35] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(124) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#124, d_year#125] + +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] + +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] + +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] + +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] + +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) + +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] + +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] + +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner + +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(113) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] - -(126) Filter [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] -Condition : ((isnotnull(d_year#125) AND (d_year#125 = 2000)) AND isnotnull(d_date_sk#124)) - -(127) BroadcastExchange -Input [2]: [d_date_sk#124, d_year#125] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -(128) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#124] -Join type: Inner -Join condition: None - -(129) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#124, d_year#125] - -(130) ReusedExchange [Reuses operator id: 44] -Output [3]: [s_store_sk#126, s_store_name#127, s_zip#128] - -(131) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#126] -Join type: Inner -Join condition: None - -(132) Project [codegen id : 51] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_sk#126, s_store_name#127, s_zip#128] - -(133) ReusedExchange [Reuses operator id: 50] -Output [6]: [c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(134) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#129] -Join type: Inner -Join condition: None - -(135) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(136) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#135, d_year#136] - -(137) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_sales_date_sk#134] -Right keys [1]: [d_date_sk#135] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134, d_date_sk#135, d_year#136] - -(139) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#137, d_year#138] - -(140) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_shipto_date_sk#133] -Right keys [1]: [d_date_sk#137] -Join type: Inner -Join condition: None - -(141) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136, d_date_sk#137, d_year#138] - -(142) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#139, cd_marital_status#140] - -(143) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#139] -Join type: Inner -Join condition: None - -(144) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_demo_sk#139, cd_marital_status#140] - -(145) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] - -(146) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_cdemo_sk#130] -Right keys [1]: [cd_demo_sk#141] -Join type: Inner -Join condition: NOT (cd_marital_status#140 = cd_marital_status#142) - -(147) Project [codegen id : 51] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140, cd_demo_sk#141, cd_marital_status#142] - -(148) ReusedExchange [Reuses operator id: 74] -Output [1]: [p_promo_sk#143] - -(149) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#143] -Join type: Inner -Join condition: None - -(150) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, p_promo_sk#143] - -(151) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#144, hd_income_band_sk#145] - -(152) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#144] -Join type: Inner -Join condition: None - -(153) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_demo_sk#144, hd_income_band_sk#145] - -(154) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] - -(155) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_hdemo_sk#131] -Right keys [1]: [hd_demo_sk#146] -Join type: Inner -Join condition: None - -(156) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_demo_sk#146, hd_income_band_sk#147] - -(157) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(158) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#148] -Join type: Inner -Join condition: None - -(159) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(160) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(161) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_addr_sk#132] -Right keys [1]: [ca_address_sk#153] -Join type: Inner -Join condition: None - -(162) Project [codegen id : 51] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(163) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#158] - -(164) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#145] -Right keys [1]: [ib_income_band_sk#158] -Join type: Inner -Join condition: None - -(165) Project [codegen id : 51] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#158] - -(166) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#159] - -(167) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#159] -Join type: Inner -Join condition: None - -(168) Project [codegen id : 51] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#159] - -(169) ReusedExchange [Reuses operator id: 108] -Output [2]: [i_item_sk#160, i_product_name#161] - -(170) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#160] -Join type: Inner -Join condition: None - -(171) Project [codegen id : 51] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] - -(172) HashAggregate [codegen id : 51] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#162, sum#163, sum#164] -Results [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] - -(173) HashAggregate [codegen id : 51] -Input [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#160 AS item_sk#168, s_store_name#127 AS store_name#169, s_zip#128 AS store_zip#170, d_year#125 AS syear#171, count(1)#84 AS cnt#172, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#173, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#174, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#175] - -(174) Exchange -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: hashpartitioning(item_sk#168, store_name#169, store_zip#170, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(175) Sort [codegen id : 52] -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: [item_sk#168 ASC NULLS FIRST, store_name#169 ASC NULLS FIRST, store_zip#170 ASC NULLS FIRST], false, 0 - -(176) SortMergeJoin [codegen id : 53] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#168, store_name#169, store_zip#170] -Join type: Inner -Join condition: (cnt#172 <= cnt#101) - -(177) Project [codegen id : 53] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] - -(178) Exchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, s1#102 ASC NULLS FIRST, s1#173 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(179) Sort [codegen id : 54] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, s1#102 ASC NULLS FIRST, s1#173 ASC NULLS FIRST], true, 0 +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) + +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] + +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight + +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] + +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] + +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight + +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] + +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight + +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] + +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight + +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] + +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] + +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight + +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] + +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight + +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] + +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] + +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] + +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight + +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] + +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight + +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] + +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] + +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight + +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] + +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] + +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] + +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] + +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) + +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] + +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST] + +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/extended.txt new file mode 100644 index 0000000000..ae10068c4c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/extended.txt @@ -0,0 +1,237 @@ +CometColumnarToRow ++- CometSort + +- CometExchange + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometProject + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.income_band + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometProject + : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : :- CometProject + : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : :- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : : : : : : : : :- CometBroadcastExchange + : : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometHashAggregate + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometSortMergeJoin + : : : : : : : : : : : : : : : :- CometSort + : : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : : : : : : : +- CometSort + : : : : : : : : : : : : : : : +- CometExchange + : : : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : : +- CometProject + : : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : : +- CometFilter + : : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : : +- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometProject + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometProject + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.promotion + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer_address + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.income_band + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.income_band + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 234 out of 234 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/simplified.txt index c010e0aea0..cda6c1a5bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/simplified.txt @@ -1,270 +1,170 @@ -WholeStageCodegen (54) - Sort [product_name,store_name,cnt,s1,s1] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [product_name,store_name,cnt,s1,s1] #1 - WholeStageCodegen (53) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (26) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [ss_item_sk] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (9) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (8) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #6 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #7 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk] - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (52) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (35) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (36) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [p_promo_sk] #13 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 + CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometExchange [product_name,store_name,cnt,s1,s1] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #2 + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #8 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #11 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 + CometFilter [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 + CometFilter [hd_demo_sk,hd_income_band_sk] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometNativeScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 + CometFilter [ib_income_band_sk] + CometNativeScan parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #18 + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/explain.txt index 3a48ca867b..b4cf1d8244 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/explain.txt @@ -1,441 +1,393 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- Window (69) - +- * Sort (68) - +- Exchange (67) - +- Union (66) - :- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.item (17) - :- * HashAggregate (30) - : +- Exchange (29) - : +- * HashAggregate (28) - : +- * HashAggregate (27) - : +- ReusedExchange (26) - :- * HashAggregate (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- * HashAggregate (32) - : +- ReusedExchange (31) - :- * HashAggregate (40) - : +- Exchange (39) - : +- * HashAggregate (38) - : +- * HashAggregate (37) - : +- ReusedExchange (36) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * HashAggregate (42) - : +- ReusedExchange (41) - :- * HashAggregate (50) - : +- Exchange (49) - : +- * HashAggregate (48) - : +- * HashAggregate (47) - : +- ReusedExchange (46) - :- * HashAggregate (55) - : +- Exchange (54) - : +- * HashAggregate (53) - : +- * HashAggregate (52) - : +- ReusedExchange (51) - :- * HashAggregate (60) - : +- Exchange (59) - : +- * HashAggregate (58) - : +- * HashAggregate (57) - : +- ReusedExchange (56) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * HashAggregate (62) - +- ReusedExchange (61) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (70) ++- * Filter (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometUnion (64) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometNativeScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometNativeScan parquet spark_catalog.default.item (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + :- CometHashAggregate (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- CometHashAggregate (30) + : +- ReusedExchange (29) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometHashAggregate (35) + : +- ReusedExchange (34) + :- CometHashAggregate (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometHashAggregate (40) + : +- ReusedExchange (39) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometHashAggregate (45) + : +- ReusedExchange (44) + :- CometHashAggregate (53) + : +- CometExchange (52) + : +- CometHashAggregate (51) + : +- CometHashAggregate (50) + : +- ReusedExchange (49) + :- CometHashAggregate (58) + : +- CometExchange (57) + : +- CometHashAggregate (56) + : +- CometHashAggregate (55) + : +- ReusedExchange (54) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometHashAggregate (60) + +- ReusedExchange (59) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +(5) CometProject Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +(8) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(14) BroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] -(16) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(17) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometNativeScan parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) -(19) Filter [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(20) BroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(22) Project [codegen id : 4] -Output [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(23) HashAggregate [codegen id : 4] -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] +(21) CometHashAggregate +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -(24) Exchange -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 5] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] +(23) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, cast(sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 as decimal(38,2)) AS sumsales#23] - -(26) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#24, isEmpty#25] -(27) HashAggregate [codegen id : 10] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#24, isEmpty#25] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(28) HashAggregate [codegen id : 10] -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sumsales#26] -Keys [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] - -(29) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(30) HashAggregate [codegen id : 11] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] -Keys [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#31] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null AS s_store_id#32, sum(sumsales#26)#31 AS sumsales#33] - -(31) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#34, isEmpty#35] - -(32) HashAggregate [codegen id : 16] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#34, isEmpty#35] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(33) HashAggregate [codegen id : 16] -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sumsales#26] -Keys [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#36, isEmpty#37] -Results [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] - -(34) Exchange -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) HashAggregate [codegen id : 17] -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] -Keys [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#40] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null AS d_moy#41, null AS s_store_id#42, sum(sumsales#26)#40 AS sumsales#43] - -(36) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#44, isEmpty#45] - -(37) HashAggregate [codegen id : 22] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#44, isEmpty#45] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(38) HashAggregate [codegen id : 22] -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sumsales#26] -Keys [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#46, isEmpty#47] -Results [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] - -(39) Exchange -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(40) HashAggregate [codegen id : 23] -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] -Keys [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#50] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null AS d_qoy#51, null AS d_moy#52, null AS s_store_id#53, sum(sumsales#26)#50 AS sumsales#54] - -(41) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#55, isEmpty#56] - -(42) HashAggregate [codegen id : 28] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#55, isEmpty#56] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(43) HashAggregate [codegen id : 28] -Input [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sumsales#26] -Keys [4]: [i_category#16, i_class#15, i_brand#14, i_product_name#17] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#57, isEmpty#58] -Results [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] - -(44) Exchange -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 29] -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] -Keys [4]: [i_category#16, i_class#15, i_brand#14, i_product_name#17] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#61] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, null AS d_year#62, null AS d_qoy#63, null AS d_moy#64, null AS s_store_id#65, sum(sumsales#26)#61 AS sumsales#66] - -(46) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#67, isEmpty#68] - -(47) HashAggregate [codegen id : 34] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#67, isEmpty#68] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [4]: [i_category#16, i_class#15, i_brand#14, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(48) HashAggregate [codegen id : 34] -Input [4]: [i_category#16, i_class#15, i_brand#14, sumsales#26] -Keys [3]: [i_category#16, i_class#15, i_brand#14] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#69, isEmpty#70] -Results [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] - -(49) Exchange -Input [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(50) HashAggregate [codegen id : 35] -Input [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] -Keys [3]: [i_category#16, i_class#15, i_brand#14] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#73] -Results [9]: [i_category#16, i_class#15, i_brand#14, null AS i_product_name#74, null AS d_year#75, null AS d_qoy#76, null AS d_moy#77, null AS s_store_id#78, sum(sumsales#26)#73 AS sumsales#79] - -(51) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#80, isEmpty#81] - -(52) HashAggregate [codegen id : 40] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#80, isEmpty#81] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [3]: [i_category#16, i_class#15, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(53) HashAggregate [codegen id : 40] -Input [3]: [i_category#16, i_class#15, sumsales#26] -Keys [2]: [i_category#16, i_class#15] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#82, isEmpty#83] -Results [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] - -(54) Exchange -Input [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] -Arguments: hashpartitioning(i_category#16, i_class#15, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(55) HashAggregate [codegen id : 41] -Input [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] -Keys [2]: [i_category#16, i_class#15] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#86] -Results [9]: [i_category#16, i_class#15, null AS i_brand#87, null AS i_product_name#88, null AS d_year#89, null AS d_qoy#90, null AS d_moy#91, null AS s_store_id#92, sum(sumsales#26)#86 AS sumsales#93] - -(56) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#94, isEmpty#95] - -(57) HashAggregate [codegen id : 46] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#94, isEmpty#95] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [2]: [i_category#16, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(58) HashAggregate [codegen id : 46] -Input [2]: [i_category#16, sumsales#26] -Keys [1]: [i_category#16] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#96, isEmpty#97] -Results [3]: [i_category#16, sum#98, isEmpty#99] - -(59) Exchange -Input [3]: [i_category#16, sum#98, isEmpty#99] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(60) HashAggregate [codegen id : 47] -Input [3]: [i_category#16, sum#98, isEmpty#99] -Keys [1]: [i_category#16] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#100] -Results [9]: [i_category#16, null AS i_class#101, null AS i_brand#102, null AS i_product_name#103, null AS d_year#104, null AS d_qoy#105, null AS d_moy#106, null AS s_store_id#107, sum(sumsales#26)#100 AS sumsales#108] - -(61) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#109, isEmpty#110] - -(62) HashAggregate [codegen id : 52] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#109, isEmpty#110] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(63) HashAggregate [codegen id : 52] -Input [1]: [sumsales#26] +(24) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] + +(25) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] + +(26) CometHashAggregate +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sumsales#32] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [partial_sum(sumsales#32)] + +(27) CometExchange +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometHashAggregate +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [sum(sumsales#32)] + +(29) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] + +(30) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#40 * cast(ss_quantity#41 as decimal(10,0))), 0.00))] + +(31) CometHashAggregate +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sumsales#32] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [partial_sum(sumsales#32)] + +(32) CometExchange +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#42, isEmpty#43] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(33) CometHashAggregate +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#42, isEmpty#43] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [sum(sumsales#32)] + +(34) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13, sum#47, isEmpty#48] + +(35) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13, sum#47, isEmpty#48] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#49 * cast(ss_quantity#50 as decimal(10,0))), 0.00))] + +(36) CometHashAggregate +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sumsales#32] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44] +Functions [1]: [partial_sum(sumsales#32)] + +(37) CometExchange +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sum#51, isEmpty#52] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(38) CometHashAggregate +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sum#51, isEmpty#52] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44] +Functions [1]: [sum(sumsales#32)] + +(39) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13, sum#56, isEmpty#57] + +(40) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13, sum#56, isEmpty#57] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#58 * cast(ss_quantity#59 as decimal(10,0))), 0.00))] + +(41) CometHashAggregate +Input [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sumsales#32] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [partial_sum(sumsales#32)] + +(42) CometExchange +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#60, isEmpty#61] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(43) CometHashAggregate +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#60, isEmpty#61] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [sum(sumsales#32)] + +(44) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13, sum#65, isEmpty#66] + +(45) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13, sum#65, isEmpty#66] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#67 * cast(ss_quantity#68 as decimal(10,0))), 0.00))] + +(46) CometHashAggregate +Input [4]: [i_category#21, i_class#20, i_brand#19, sumsales#32] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [partial_sum(sumsales#32)] + +(47) CometExchange +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#69, isEmpty#70] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(48) CometHashAggregate +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#69, isEmpty#70] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [sum(sumsales#32)] + +(49) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13, sum#74, isEmpty#75] + +(50) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13, sum#74, isEmpty#75] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#76 * cast(ss_quantity#77 as decimal(10,0))), 0.00))] + +(51) CometHashAggregate +Input [3]: [i_category#21, i_class#20, sumsales#32] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [partial_sum(sumsales#32)] + +(52) CometExchange +Input [4]: [i_category#21, i_class#20, sum#78, isEmpty#79] +Arguments: hashpartitioning(i_category#21, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(53) CometHashAggregate +Input [4]: [i_category#21, i_class#20, sum#78, isEmpty#79] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [sum(sumsales#32)] + +(54) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13, sum#83, isEmpty#84] + +(55) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13, sum#83, isEmpty#84] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#85 * cast(ss_quantity#86 as decimal(10,0))), 0.00))] + +(56) CometHashAggregate +Input [2]: [i_category#21, sumsales#32] +Keys [1]: [i_category#21] +Functions [1]: [partial_sum(sumsales#32)] + +(57) CometExchange +Input [3]: [i_category#21, sum#87, isEmpty#88] +Arguments: hashpartitioning(i_category#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(58) CometHashAggregate +Input [3]: [i_category#21, sum#87, isEmpty#88] +Keys [1]: [i_category#21] +Functions [1]: [sum(sumsales#32)] + +(59) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13, sum#92, isEmpty#93] + +(60) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13, sum#92, isEmpty#93] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#94 * cast(ss_quantity#95 as decimal(10,0))), 0.00))] + +(61) CometHashAggregate +Input [1]: [sumsales#32] Keys: [] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#111, isEmpty#112] -Results [2]: [sum#113, isEmpty#114] +Functions [1]: [partial_sum(sumsales#32)] -(64) Exchange -Input [2]: [sum#113, isEmpty#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(62) CometExchange +Input [2]: [sum#96, isEmpty#97] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(65) HashAggregate [codegen id : 53] -Input [2]: [sum#113, isEmpty#114] +(63) CometHashAggregate +Input [2]: [sum#96, isEmpty#97] Keys: [] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#115] -Results [9]: [null AS i_category#116, null AS i_class#117, null AS i_brand#118, null AS i_product_name#119, null AS d_year#120, null AS d_qoy#121, null AS d_moy#122, null AS s_store_id#123, sum(sumsales#26)#115 AS sumsales#124] - -(66) Union - -(67) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(68) Sort [codegen id : 54] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: [i_category#16 ASC NULLS FIRST, sumsales#23 DESC NULLS LAST], false, 0 - -(69) Window -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: [rank(sumsales#23) windowspecdefinition(i_category#16, sumsales#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#125], [i_category#16], [sumsales#23 DESC NULLS LAST] - -(70) Filter [codegen id : 55] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] -Condition : (rk#125 <= 100) - -(71) TakeOrderedAndProject -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] -Arguments: 100, [i_category#16 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, d_moy#9 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sumsales#23 ASC NULLS FIRST, rk#125 ASC NULLS FIRST], [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] +Functions [1]: [sum(sumsales#32)] + +(64) CometUnion +Child 0 Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Child 1 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#107, sumsales#108] +Child 2 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#109, s_store_id#110, sumsales#111] +Child 3 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#112, d_moy#113, s_store_id#114, sumsales#115] +Child 4 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#116, d_qoy#117, d_moy#118, s_store_id#119, sumsales#120] +Child 5 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#121, d_year#122, d_qoy#123, d_moy#124, s_store_id#125, sumsales#126] +Child 6 Input [9]: [i_category#21, i_class#20, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sumsales#133] +Child 7 Input [9]: [i_category#21, i_class#134, i_brand#135, i_product_name#136, d_year#137, d_qoy#138, d_moy#139, s_store_id#140, sumsales#141] +Child 8 Input [9]: [i_category#142, i_class#143, i_brand#144, i_product_name#145, d_year#146, d_qoy#147, d_moy#148, s_store_id#149, sumsales#150] + +(65) CometExchange +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: hashpartitioning(i_category#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(66) CometSort +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106], [i_category#98 ASC NULLS FIRST, sumsales#106 DESC NULLS LAST] + +(67) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] + +(68) Window +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: [rank(sumsales#106) windowspecdefinition(i_category#98, sumsales#106 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#151], [i_category#98], [sumsales#106 DESC NULLS LAST] + +(69) Filter [codegen id : 2] +Input [10]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] +Condition : (rk#151 <= 100) + +(70) TakeOrderedAndProject +Input [10]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] +Arguments: 100, [i_category#98 ASC NULLS FIRST, i_class#99 ASC NULLS FIRST, i_brand#100 ASC NULLS FIRST, i_product_name#101 ASC NULLS FIRST, d_year#102 ASC NULLS FIRST, d_qoy#103 ASC NULLS FIRST, d_moy#104 ASC NULLS FIRST, s_store_id#105 ASC NULLS FIRST, sumsales#106 ASC NULLS FIRST, rk#151 ASC NULLS FIRST], [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/extended.txt new file mode 100644 index 0000000000..f637d4da8b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/extended.txt @@ -0,0 +1,240 @@ + TakeOrderedAndProject [COMET: ] ++- Filter + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 234 out of 237 eligible operators (98%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/simplified.txt index 51f848bf7b..bb081ddfb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/simplified.txt @@ -1,120 +1,74 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (55) + WholeStageCodegen (2) Filter [rk] InputAdapter Window [sumsales,i_category] - WholeStageCodegen (54) - Sort [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - Union - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - WholeStageCodegen (11) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 - WholeStageCodegen (10) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] [sum(sumsales),d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 - WholeStageCodegen (16) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (23) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] [sum(sumsales),d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year] #8 - WholeStageCodegen (22) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (29) - HashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty] [sum(sumsales),d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name] #9 - WholeStageCodegen (28) - HashAggregate [i_category,i_class,i_brand,i_product_name,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (35) - HashAggregate [i_category,i_class,i_brand,sum,isEmpty] [sum(sumsales),i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand] #10 - WholeStageCodegen (34) - HashAggregate [i_category,i_class,i_brand,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (41) - HashAggregate [i_category,i_class,sum,isEmpty] [sum(sumsales),i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class] #11 - WholeStageCodegen (40) - HashAggregate [i_category,i_class,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (47) - HashAggregate [i_category,sum,isEmpty] [sum(sumsales),i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category] #12 - WholeStageCodegen (46) - HashAggregate [i_category,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (53) - HashAggregate [sum,isEmpty] [sum(sumsales),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange #13 - WholeStageCodegen (52) - HashAggregate [sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometExchange [i_category] #1 + CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] + CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] + CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] + CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name] #9 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] + CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand] #10 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] + CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class] #11 + CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] + CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category] #12 + CometHashAggregate [sumsales] [i_category,sum,isEmpty] + CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange #13 + CometHashAggregate [sumsales] [sum,isEmpty] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/explain.txt index e0548b3cc1..9968808879 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/explain.txt @@ -1,351 +1,368 @@ == Physical Plan == -TakeOrderedAndProject (59) -+- * Project (58) - +- Window (57) - +- * Sort (56) - +- Exchange (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- Union (51) - :- * HashAggregate (40) - : +- Exchange (39) - : +- * HashAggregate (38) - : +- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (35) - : +- * BroadcastHashJoin LeftSemi BuildRight (34) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (33) - : +- * Project (32) - : +- * Filter (31) - : +- Window (30) - : +- * Sort (29) - : +- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Filter (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.store_sales (14) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.store (17) - : +- ReusedExchange (23) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * HashAggregate (42) - : +- ReusedExchange (41) - +- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * HashAggregate (47) - +- ReusedExchange (46) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (65) ++- * Project (64) + +- Window (63) + +- * CometColumnarToRow (62) + +- CometSort (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometColumnarExchange (58) + +- * HashAggregate (57) + +- Union (56) + :- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometColumnarExchange (39) + : +- * HashAggregate (38) + : +- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * CometColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (35) + : +- * Project (34) + : +- * BroadcastHashJoin LeftSemi BuildRight (33) + : :- * CometColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometNativeScan parquet spark_catalog.default.store (10) + : +- BroadcastExchange (32) + : +- * Project (31) + : +- * Filter (30) + : +- Window (29) + : +- * CometColumnarToRow (28) + : +- CometSort (27) + : +- CometHashAggregate (26) + : +- CometExchange (25) + : +- CometHashAggregate (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (14) + : : : +- CometNativeScan parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometNativeScan parquet spark_catalog.default.store (15) + : +- ReusedExchange (21) + :- * HashAggregate (48) + : +- * CometColumnarToRow (47) + : +- CometColumnarExchange (46) + : +- * HashAggregate (45) + : +- * HashAggregate (44) + : +- * CometColumnarToRow (43) + : +- ReusedExchange (42) + +- * HashAggregate (55) + +- * CometColumnarToRow (54) + +- CometColumnarExchange (53) + +- * HashAggregate (52) + +- * HashAggregate (51) + +- * CometColumnarToRow (50) + +- ReusedExchange (49) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] +(8) CometProject Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(11) Scan parquet spark_catalog.default.store +(9) CometColumnarToRow [codegen id : 4] +Input [2]: [ss_store_sk#1, ss_net_profit#2] + +(10) CometNativeScan parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 7] +(11) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) -(13) Filter [codegen id : 7] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) -(14) Scan parquet spark_catalog.default.store_sales +(13) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(16) Filter [codegen id : 4] +(14) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(17) Scan parquet spark_catalog.default.store +(15) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] - -(19) Filter [codegen id : 2] +(16) CometFilter Input [2]: [s_store_sk#12, s_state#13] Condition : isnotnull(s_store_sk#12) -(20) BroadcastExchange +(17) CometProject Input [2]: [s_store_sk#12, s_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#9] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] -(22) Project [codegen id : 4] -Output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] +(19) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#14] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#11] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight -(25) Project [codegen id : 4] -Output [2]: [ss_net_profit#10, s_state#13] -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] -(26) HashAggregate [codegen id : 4] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] -(27) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] -(29) Sort [codegen id : 5] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 +(27) CometSort +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(30) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#14, _w0#17, s_state#14] -(31) Filter [codegen id : 6] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] -Condition : (ranking#19 <= 5) +(29) Window +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] -(32) Project [codegen id : 6] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] +(30) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) -(33) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] +(31) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(32) BroadcastExchange +Input [1]: [s_state#14] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] + +(33) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None -(35) BroadcastExchange +(34) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(36) BroadcastHashJoin [codegen id : 8] +(35) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(36) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(37) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_county#7, s_state#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(37) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_county#7, s_state#19] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] -(38) HashAggregate [codegen id : 8] -Input [3]: [ss_net_profit#2, s_county#7, s_state#8] -Keys [2]: [s_state#8, s_county#7] +(38) HashAggregate [codegen id : 4] +Input [3]: [ss_net_profit#2, s_county#7, s_state#19] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#20] -Results [3]: [s_state#8, s_county#7, sum#21] +Results [3]: [s_state#19, s_county#7, sum#21] -(39) Exchange -Input [3]: [s_state#8, s_county#7, sum#21] -Arguments: hashpartitioning(s_state#8, s_county#7, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(39) CometColumnarExchange +Input [3]: [s_state#19, s_county#7, sum#21] +Arguments: hashpartitioning(s_state#19, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 9] -Input [3]: [s_state#8, s_county#7, sum#21] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#8, s_county#7, 0 AS g_state#24, 0 AS g_county#25, 0 AS lochierarchy#26] +(40) CometColumnarToRow [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] -(41) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#8, s_county#7, sum#27] - -(42) HashAggregate [codegen id : 18] -Input [3]: [s_state#8, s_county#7, sum#27] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28, s_state#8] - -(43) HashAggregate [codegen id : 18] -Input [2]: [total_sum#28, s_state#8] -Keys [1]: [s_state#8] -Functions [1]: [partial_sum(total_sum#28)] -Aggregate Attributes [2]: [sum#29, isEmpty#30] -Results [3]: [s_state#8, sum#31, isEmpty#32] - -(44) Exchange -Input [3]: [s_state#8, sum#31, isEmpty#32] -Arguments: hashpartitioning(s_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 19] -Input [3]: [s_state#8, sum#31, isEmpty#32] -Keys [1]: [s_state#8] -Functions [1]: [sum(total_sum#28)] -Aggregate Attributes [1]: [sum(total_sum#28)#33] -Results [6]: [sum(total_sum#28)#33 AS total_sum#34, s_state#8, null AS s_county#35, 0 AS g_state#36, 1 AS g_county#37, 1 AS lochierarchy#38] - -(46) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#8, s_county#7, sum#39] - -(47) HashAggregate [codegen id : 28] -Input [3]: [s_state#8, s_county#7, sum#39] -Keys [2]: [s_state#8, s_county#7] +(41) HashAggregate [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28] - -(48) HashAggregate [codegen id : 28] -Input [1]: [total_sum#28] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#19 AS s_state#24, s_county#7 AS s_county#25, 0 AS g_state#26, 0 AS g_county#27, 0 AS lochierarchy#28] + +(42) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_state#19, s_county#29, sum#30] + +(43) CometColumnarToRow [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] + +(44) HashAggregate [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] +Keys [2]: [s_state#19, s_county#29] +Functions [1]: [sum(UnscaledValue(ss_net_profit#31))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#31))#22] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#31))#22,17,2) AS total_sum#32, s_state#19] + +(45) HashAggregate [codegen id : 10] +Input [2]: [total_sum#32, s_state#19] +Keys [1]: [s_state#19] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#33, isEmpty#34] +Results [3]: [s_state#19, sum#35, isEmpty#36] + +(46) CometColumnarExchange +Input [3]: [s_state#19, sum#35, isEmpty#36] +Arguments: hashpartitioning(s_state#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(47) CometColumnarToRow [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] + +(48) HashAggregate [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Keys [1]: [s_state#19] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#37] +Results [6]: [sum(total_sum#32)#37 AS total_sum#38, s_state#19, null AS s_county#39, 0 AS g_state#40, 1 AS g_county#41, 1 AS lochierarchy#42] + +(49) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_state#19, s_county#43, sum#44] + +(50) CometColumnarToRow [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] + +(51) HashAggregate [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] +Keys [2]: [s_state#19, s_county#43] +Functions [1]: [sum(UnscaledValue(ss_net_profit#45))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#45))#22] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#45))#22,17,2) AS total_sum#32] + +(52) HashAggregate [codegen id : 16] +Input [1]: [total_sum#32] Keys: [] -Functions [1]: [partial_sum(total_sum#28)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [2]: [sum#42, isEmpty#43] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#46, isEmpty#47] +Results [2]: [sum#48, isEmpty#49] -(49) Exchange -Input [2]: [sum#42, isEmpty#43] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(53) CometColumnarExchange +Input [2]: [sum#48, isEmpty#49] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(50) HashAggregate [codegen id : 29] -Input [2]: [sum#42, isEmpty#43] +(54) CometColumnarToRow [codegen id : 17] +Input [2]: [sum#48, isEmpty#49] + +(55) HashAggregate [codegen id : 17] +Input [2]: [sum#48, isEmpty#49] Keys: [] -Functions [1]: [sum(total_sum#28)] -Aggregate Attributes [1]: [sum(total_sum#28)#44] -Results [6]: [sum(total_sum#28)#44 AS total_sum#45, null AS s_state#46, null AS s_county#47, 1 AS g_state#48, 1 AS g_county#49, 2 AS lochierarchy#50] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#50] +Results [6]: [sum(total_sum#32)#50 AS total_sum#51, null AS s_state#52, null AS s_county#53, 1 AS g_state#54, 1 AS g_county#55, 2 AS lochierarchy#56] -(51) Union +(56) Union -(52) HashAggregate [codegen id : 30] -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Keys [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] +(57) HashAggregate [codegen id : 18] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] +Results [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] -(53) Exchange -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Arguments: hashpartitioning(total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(58) CometColumnarExchange +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Arguments: hashpartitioning(total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(54) HashAggregate [codegen id : 31] -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Keys [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] +(59) CometHashAggregate +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, CASE WHEN (g_county#25 = 0) THEN s_state#8 END AS _w0#51] -(55) Exchange -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: hashpartitioning(lochierarchy#26, _w0#51, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(60) CometExchange +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: hashpartitioning(lochierarchy#28, _w0#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(61) CometSort +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57], [lochierarchy#28 ASC NULLS FIRST, _w0#57 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] -(56) Sort [codegen id : 32] -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: [lochierarchy#26 ASC NULLS FIRST, _w0#51 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST], false, 0 +(62) CometColumnarToRow [codegen id : 19] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] -(57) Window -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#26, _w0#51, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#52], [lochierarchy#26, _w0#51], [total_sum#23 DESC NULLS LAST] +(63) Window +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#28, _w0#57, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#58], [lochierarchy#28, _w0#57], [total_sum#23 DESC NULLS LAST] -(58) Project [codegen id : 33] -Output [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] -Input [6]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51, rank_within_parent#52] +(64) Project [codegen id : 20] +Output [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] +Input [6]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57, rank_within_parent#58] -(59) TakeOrderedAndProject -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] -Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN s_state#8 END ASC NULLS FIRST, rank_within_parent#52 ASC NULLS FIRST], [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] +(65) TakeOrderedAndProject +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#24 END ASC NULLS FIRST, rank_within_parent#58 ASC NULLS FIRST], [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/extended.txt new file mode 100644 index 0000000000..0da4471642 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/extended.txt @@ -0,0 +1,152 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Filter + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- BroadcastExchange + : +- Project + : +- BroadcastHashJoin + : :- CometColumnarToRow + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- Project + : +- Filter + : +- Window [COMET: Partitioning and sorting specifications must be the same.] + : +- CometColumnarToRow + : +- CometSort + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- BroadcastExchange + +- Project + +- BroadcastHashJoin + :- CometColumnarToRow + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- BroadcastExchange + +- Project + +- Filter + +- Window [COMET: Partitioning and sorting specifications must be the same.] + +- CometColumnarToRow + +- CometSort + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 93 out of 135 eligible operators (68%). Final plan contains 15 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/simplified.txt index 4a20e8104c..5a3d152ba5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/simplified.txt @@ -1,96 +1,91 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (33) + WholeStageCodegen (20) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (32) - Sort [lochierarchy,_w0,total_sum] + WholeStageCodegen (19) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (31) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - WholeStageCodegen (30) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] - InputAdapter - Union - WholeStageCodegen (9) - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] + CometSort [total_sum,s_state,s_county,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_state] [total_sum,s_state,s_county,lochierarchy,_w0,g_county] + CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 + WholeStageCodegen (18) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum] + CometColumnarToRow InputAdapter - Exchange [s_state,s_county] #3 - WholeStageCodegen (8) + CometColumnarExchange [s_state,s_county] #3 + WholeStageCodegen (4) HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] Project [ss_net_profit,s_county,s_state] BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 - WholeStageCodegen (7) - BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WholeStageCodegen (5) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + CometFilter [s_store_sk,s_county,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [s_state] #7 - WholeStageCodegen (4) - HashAggregate [s_state,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_state] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_net_profit,ss_sold_date_sk,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #7 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #8 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #4 - WholeStageCodegen (19) - HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + WholeStageCodegen (11) + HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [s_state] #9 - WholeStageCodegen (18) + CometColumnarExchange [s_state] #9 + WholeStageCodegen (10) HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 - WholeStageCodegen (29) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange #10 - WholeStageCodegen (28) + CometColumnarExchange #10 + WholeStageCodegen (16) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/explain.txt index 9d2596a538..ce0e312071 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/explain.txt @@ -1,423 +1,369 @@ == Physical Plan == -TakeOrderedAndProject (74) -+- * HashAggregate (73) - +- Exchange (72) - +- * HashAggregate (71) - +- * Project (70) - +- * SortMergeJoin LeftOuter (69) - :- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * BroadcastHashJoin LeftOuter BuildRight (59) - : :- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (42) - : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : :- * Project (35) - : : : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : : : :- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Project (15) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : : : : :- * Project (9) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) - : : : : : : : : : +- Scan parquet spark_catalog.default.inventory (4) - : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) - : : : : : : : : +- Scan parquet spark_catalog.default.warehouse (10) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (22) - : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) - : : : : : +- Scan parquet spark_catalog.default.household_demographics (29) - : : : : +- BroadcastExchange (40) - : : : : +- * Project (39) - : : : : +- * Filter (38) - : : : : +- * ColumnarToRow (37) - : : : : +- Scan parquet spark_catalog.default.date_dim (36) - : : : +- BroadcastExchange (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) - : : : +- Scan parquet spark_catalog.default.date_dim (43) - : : +- BroadcastExchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.date_dim (49) - : +- BroadcastExchange (58) - : +- * Filter (57) - : +- * ColumnarToRow (56) - : +- Scan parquet spark_catalog.default.promotion (55) - +- * Sort (68) - +- Exchange (67) - +- * Project (66) - +- * Filter (65) - +- * ColumnarToRow (64) - +- Scan parquet spark_catalog.default.catalog_returns (63) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometNativeScan parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometNativeScan parquet spark_catalog.default.catalog_returns (53) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 10] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.inventory +(3) CometNativeScan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -(8) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [inv_item_sk#9] -Join type: Inner -Join condition: (inv_quantity_on_hand#11 < cs_quantity#7) +(6) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight -(9) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +(7) CometProject Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometNativeScan parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] Condition : isnotnull(w_warehouse_sk#13) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] -(14) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [inv_warehouse_sk#10] -Right keys [1]: [w_warehouse_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight -(15) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +(12) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -(16) Scan parquet spark_catalog.default.item +(13) CometNativeScan parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [2]: [i_item_sk#15, i_item_desc#16] Condition : isnotnull(i_item_sk#15) -(19) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [i_item_sk#15, i_item_desc#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_desc#16] -(20) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight -(21) Project [codegen id : 10] -Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(17) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(22) Scan parquet spark_catalog.default.customer_demographics +(18) CometNativeScan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,M), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(24) Filter [codegen id : 4] +(19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = M)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#17)) -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#17] +(20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] -(26) BroadcastExchange +(21) CometBroadcastExchange Input [1]: [cd_demo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [cd_demo_sk#17] -(27) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight -(28) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(23) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(29) Scan parquet spark_catalog.default.household_demographics +(24) CometNativeScan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(31) Filter [codegen id : 5] +(25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#19)) -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#19] +(26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] -(33) BroadcastExchange +(27) CometBroadcastExchange Input [1]: [hd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [hd_demo_sk#19] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight -(35) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(29) CometProject Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(36) Scan parquet spark_catalog.default.date_dim +(30) CometNativeScan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(38) Filter [codegen id : 6] +(31) CometFilter Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) -(39) Project [codegen id : 6] -Output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +(32) CometProject Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] -(40) BroadcastExchange +(33) CometBroadcastExchange Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight -(42) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(35) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(43) Scan parquet spark_catalog.default.date_dim +(36) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] - -(45) Filter [codegen id : 7] +(37) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(46) BroadcastExchange +(38) CometBroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [plan_id=7] +Arguments: [d_date_sk#25, d_week_seq#26] -(47) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [d_week_seq#23, inv_date_sk#12] -Right keys [2]: [d_week_seq#26, d_date_sk#25] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight -(48) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(40) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(49) Scan parquet spark_catalog.default.date_dim +(41) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] - -(51) Filter [codegen id : 8] +(42) CometFilter Input [2]: [d_date_sk#27, d_date#28] Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(52) BroadcastExchange +(43) CometBroadcastExchange Input [2]: [d_date_sk#27, d_date#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: [d_date_sk#27, d_date#28] -(53) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: (d_date#28 > date_add(d_date#22, 5)) +(44) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight -(54) Project [codegen id : 10] -Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(45) CometProject Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(55) Scan parquet spark_catalog.default.promotion +(46) CometNativeScan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(56) ColumnarToRow [codegen id : 9] -Input [1]: [p_promo_sk#29] - -(57) Filter [codegen id : 9] +(47) CometFilter Input [1]: [p_promo_sk#29] Condition : isnotnull(p_promo_sk#29) -(58) BroadcastExchange +(48) CometBroadcastExchange Input [1]: [p_promo_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [p_promo_sk#29] -(59) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_promo_sk#5] -Right keys [1]: [p_promo_sk#29] -Join type: LeftOuter -Join condition: None +(49) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight -(60) Project [codegen id : 10] -Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(50) CometProject Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(61) Exchange +(51) CometExchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(62) Sort [codegen id : 11] +(52) CometSort Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(63) Scan parquet spark_catalog.default.catalog_returns +(53) CometNativeScan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(64) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(65) Filter [codegen id : 12] +(54) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(66) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] +(55) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] -(67) Exchange +(56) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(68) Sort [codegen id : 13] +(57) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] -(69) SortMergeJoin [codegen id : 14] -Left keys [2]: [cs_item_sk#4, cs_order_number#6] -Right keys [2]: [cr_item_sk#30, cr_order_number#31] -Join type: LeftOuter -Join condition: None +(58) CometSortMergeJoin +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter -(70) Project [codegen id : 14] -Output [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(59) CometProject Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(71) HashAggregate [codegen id : 14] +(60) CometHashAggregate Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#33] -Results [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -(72) Exchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(61) CometExchange +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(73) HashAggregate [codegen id : 15] -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] +(62) CometHashAggregate +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#35] -Results [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count(1)#35 AS no_promo#36, count(1)#35 AS promo#37, count(1)#35 AS total_cnt#38] -(74) TakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] -Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] +(63) CometTakeOrderedAndProject +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + +(64) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/extended.txt new file mode 100644 index 0000000000..3bf43ef8c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/extended.txt @@ -0,0 +1,66 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometBroadcastHashJoin + : : : : : : :- CometProject + : : : : : : : +- CometBroadcastHashJoin + : : : : : : : :- CometProject + : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : :- CometProject + : : : : : : : : : +- CometBroadcastHashJoin + : : : : : : : : : :- CometFilter + : : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : : : : +- CometBroadcastExchange + : : : : : : : : : +- CometFilter + : : : : : : : : : +- CometNativeScan parquet spark_catalog.default.inventory + : : : : : : : : +- CometBroadcastExchange + : : : : : : : : +- CometFilter + : : : : : : : : +- CometNativeScan parquet spark_catalog.default.warehouse + : : : : : : : +- CometBroadcastExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : : : : +- CometBroadcastExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.customer_demographics + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.household_demographics + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.catalog_returns + +Comet accelerated 63 out of 63 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/simplified.txt index 37b3b330ae..073bd57a8c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/simplified.txt @@ -1,114 +1,66 @@ -TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] - WholeStageCodegen (15) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] - InputAdapter - Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - WholeStageCodegen (14) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] - Project [w_warehouse_name,i_item_desc,d_week_seq] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (11) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #2 - WholeStageCodegen (10) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk] - Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - WholeStageCodegen (13) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] + CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] + CometProject [w_warehouse_name,i_item_desc,d_week_seq] + CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometExchange [cs_item_sk,cs_order_number] #2 + CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometNativeScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometNativeScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometNativeScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometNativeScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 + CometFilter [d_date_sk,d_week_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 + CometFilter [p_promo_sk] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/explain.txt index 62c5a478de..d06220c1f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/explain.txt @@ -1,418 +1,383 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometNativeScan parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometNativeScan parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometNativeScan parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(4) CometNativeScan parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) -(6) Filter [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] -(7) BroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] - -(16) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(17) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] + +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) + +(18) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(23) ReusedExchange [Reuses operator id: 7] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#15] -Right keys [1]: [ss_customer_sk#19] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(25) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] +(24) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] - -(32) HashAggregate [codegen id : 6] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(33) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(35) BroadcastExchange +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] + +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight + +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] + +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(37) Scan parquet spark_catalog.default.customer +(34) CometNativeScan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(39) Filter [codegen id : 10] +(36) CometProject Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_customer_id#31)) +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(40) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] +(37) CometNativeScan parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(42) Filter [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_customer_sk#34) - -(43) BroadcastExchange -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [ws_bill_customer_sk#34] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36] -Input [7]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33, ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#37, d_year#38] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#36] -Right keys [1]: [d_date_sk#37] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Input [7]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36, d_date_sk#37, d_year#38] - -(49) HashAggregate [codegen id : 10] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum#39] -Results [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] - -(50) Exchange -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Arguments: hashpartitioning(c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#41] -Results [2]: [c_customer_id#31 AS customer_id#42, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#41,17,2) AS year_total#43] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#42, year_total#43] -Condition : (isnotnull(year_total#43) AND (year_total#43 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#42, year_total#43] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#42] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#42, year_total#43] - -(56) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) + +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] + +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight + +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometNativeScan parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] - -(58) Filter [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(c_customer_id#45)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [3]: [ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#44] -Right keys [1]: [ws_bill_customer_sk#48] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50] -Input [7]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#51, d_year#52] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Input [7]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(65) HashAggregate [codegen id : 14] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum#53] -Results [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] - -(66) Exchange -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#49))#41] -Results [2]: [c_customer_id#45 AS customer_id#55, MakeDecimal(sum(UnscaledValue(ws_net_paid#49))#41,17,2) AS year_total#56] - -(68) BroadcastExchange +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] + +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] + +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight + +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] + +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] + +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] + +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] + +(64) CometBroadcastExchange Input [2]: [customer_id#55, year_total#56] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] +Arguments: [customer_id#55, year_total#56] -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#55] -Join type: Inner -Join condition: (CASE WHEN (year_total#43 > 0.00) THEN (year_total#56 / year_total#43) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -(70) Project [codegen id : 16] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43, customer_id#55, year_total#56] +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#27 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_last_name#28 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] -(71) TakeOrderedAndProject +(68) CometColumnarToRow [codegen id : 1] Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/extended.txt new file mode 100644 index 0000000000..bcd6c3e73d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/extended.txt @@ -0,0 +1,78 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : : +- CometBroadcastExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.customer + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.customer + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.customer + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_sales + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 75 out of 75 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/simplified.txt index 8ad902e176..95b5b340e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/simplified.txt @@ -1,106 +1,70 @@ -TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] + CometProject [customer_id,customer_first_name,customer_last_name] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometNativeScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/explain.txt index bf7bc3e171..7239df4e84 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/explain.txt @@ -1,754 +1,681 @@ == Physical Plan == -TakeOrderedAndProject (135) -+- * Project (134) - +- * SortMergeJoin Inner (133) - :- * Sort (74) - : +- Exchange (73) - : +- * Filter (72) - : +- * HashAggregate (71) - : +- Exchange (70) - : +- * HashAggregate (69) - : +- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * Project (26) - : : +- * SortMergeJoin LeftOuter (25) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- * Sort (24) - : : +- Exchange (23) - : : +- * Project (22) - : : +- * Filter (21) - : : +- * ColumnarToRow (20) - : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : :- * Project (45) - : : +- * SortMergeJoin LeftOuter (44) - : : :- * Sort (37) - : : : +- Exchange (36) - : : : +- * Project (35) - : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Project (32) - : : : : +- * BroadcastHashJoin Inner BuildRight (31) - : : : : :- * Filter (29) - : : : : : +- * ColumnarToRow (28) - : : : : : +- Scan parquet spark_catalog.default.store_sales (27) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- * Sort (43) - : : +- Exchange (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.store_returns (38) - : +- * Project (64) - : +- * SortMergeJoin LeftOuter (63) - : :- * Sort (56) - : : +- Exchange (55) - : : +- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (51) - : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : :- * Filter (48) - : : : : +- * ColumnarToRow (47) - : : : : +- Scan parquet spark_catalog.default.web_sales (46) - : : : +- ReusedExchange (49) - : : +- ReusedExchange (52) - : +- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * Filter (59) - : +- * ColumnarToRow (58) - : +- Scan parquet spark_catalog.default.web_returns (57) - +- * Sort (132) - +- Exchange (131) - +- * Filter (130) - +- * HashAggregate (129) - +- Exchange (128) - +- * HashAggregate (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- Union (123) - :- * Project (92) - : +- * SortMergeJoin LeftOuter (91) - : :- * Sort (88) - : : +- Exchange (87) - : : +- * Project (86) - : : +- * BroadcastHashJoin Inner BuildRight (85) - : : :- * Project (80) - : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : :- * Filter (77) - : : : : +- * ColumnarToRow (76) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : +- ReusedExchange (78) - : : +- BroadcastExchange (84) - : : +- * Filter (83) - : : +- * ColumnarToRow (82) - : : +- Scan parquet spark_catalog.default.date_dim (81) - : +- * Sort (90) - : +- ReusedExchange (89) - :- * Project (107) - : +- * SortMergeJoin LeftOuter (106) - : :- * Sort (103) - : : +- Exchange (102) - : : +- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Filter (95) - : : : : +- * ColumnarToRow (94) - : : : : +- Scan parquet spark_catalog.default.store_sales (93) - : : : +- ReusedExchange (96) - : : +- ReusedExchange (99) - : +- * Sort (105) - : +- ReusedExchange (104) - +- * Project (122) - +- * SortMergeJoin LeftOuter (121) - :- * Sort (118) - : +- Exchange (117) - : +- * Project (116) - : +- * BroadcastHashJoin Inner BuildRight (115) - : :- * Project (113) - : : +- * BroadcastHashJoin Inner BuildRight (112) - : : :- * Filter (110) - : : : +- * ColumnarToRow (109) - : : : +- Scan parquet spark_catalog.default.web_sales (108) - : : +- ReusedExchange (111) - : +- ReusedExchange (114) - +- * Sort (120) - +- ReusedExchange (119) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometProject (122) + +- CometSortMergeJoin (121) + :- CometSort (66) + : +- CometExchange (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometNativeScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometNativeScan parquet spark_catalog.default.web_returns (50) + +- CometSort (120) + +- CometExchange (119) + +- CometFilter (118) + +- CometHashAggregate (117) + +- CometExchange (116) + +- CometHashAggregate (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometProject (82) + : +- CometSortMergeJoin (81) + : :- CometSort (78) + : : +- CometExchange (77) + : : +- CometProject (76) + : : +- CometBroadcastHashJoin (75) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometFilter (68) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (67) + : : : +- ReusedExchange (69) + : : +- CometBroadcastExchange (74) + : : +- CometFilter (73) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (72) + : +- CometSort (80) + : +- ReusedExchange (79) + :- CometProject (96) + : +- CometSortMergeJoin (95) + : :- CometSort (92) + : : +- CometExchange (91) + : : +- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometFilter (84) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (83) + : : : +- ReusedExchange (85) + : : +- ReusedExchange (88) + : +- CometSort (94) + : +- ReusedExchange (93) + +- CometProject (110) + +- CometSortMergeJoin (109) + :- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometBroadcastHashJoin (103) + : :- CometProject (101) + : : +- CometBroadcastHashJoin (100) + : : :- CometFilter (98) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (97) + : : +- ReusedExchange (99) + : +- ReusedExchange (102) + +- CometSort (108) + +- ReusedExchange (107) + + +(1) CometNativeScan parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(8) CometProject Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#12, d_year#13] Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12, d_year#13] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight -(16) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +(13) CometProject Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -(17) Exchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(18) Sort [codegen id : 4] +(15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometNativeScan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(21) Filter [codegen id : 5] +(17) CometFilter Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) -(22) Project [codegen id : 5] -Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +(18) CometProject Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -(23) Exchange +(19) CometExchange Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(24) Sort [codegen id : 6] +(20) CometSort Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 7] -Left keys [2]: [cs_order_number#2, cs_item_sk#1] -Right keys [2]: [cr_order_number#15, cr_item_sk#14] -Join type: LeftOuter -Join condition: None +(21) CometSortMergeJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter -(26) Project [codegen id : 7] -Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] +(22) CometProject Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(27) Scan parquet spark_catalog.default.store_sales +(23) CometNativeScan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] - -(29) Filter [codegen id : 10] +(24) CometFilter Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_item_sk#21) -(30) ReusedExchange [Reuses operator id: 8] +(25) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(31) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_item_sk#21] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight -(32) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +(27) CometProject Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(33) ReusedExchange [Reuses operator id: 14] +(28) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#31, d_year#32] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#25] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight -(35) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +(30) CometProject Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -(36) Exchange +(31) CometExchange Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) Sort [codegen id : 11] +(32) CometSort Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] -(38) Scan parquet spark_catalog.default.store_returns +(33) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(40) Filter [codegen id : 12] +(34) CometFilter Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) -(41) Project [codegen id : 12] -Output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +(35) CometProject Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -(42) Exchange +(36) CometExchange Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(43) Sort [codegen id : 13] +(37) CometSort Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] -(44) SortMergeJoin [codegen id : 14] -Left keys [2]: [ss_ticket_number#22, ss_item_sk#21] -Right keys [2]: [sr_ticket_number#34, sr_item_sk#33] -Join type: LeftOuter -Join condition: None +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter -(45) Project [codegen id : 14] -Output [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] +(39) CometProject Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -(46) Scan parquet spark_catalog.default.web_sales +(40) CometNativeScan parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] - -(48) Filter [codegen id : 17] +(41) CometFilter Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Condition : isnotnull(ws_item_sk#40) -(49) ReusedExchange [Reuses operator id: 8] +(42) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(50) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#40] -Right keys [1]: [i_item_sk#45] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight -(51) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +(44) CometProject Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(52) ReusedExchange [Reuses operator id: 14] +(45) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#50, d_year#51] -(53) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#44] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight -(54) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +(47) CometProject Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -(55) Exchange +(48) CometExchange Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(56) Sort [codegen id : 18] +(49) CometSort Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST], false, 0 +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] -(57) Scan parquet spark_catalog.default.web_returns +(50) CometNativeScan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(59) Filter [codegen id : 19] +(51) CometFilter Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) -(60) Project [codegen id : 19] -Output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +(52) CometProject Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -(61) Exchange +(53) CometExchange Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(62) Sort [codegen id : 20] +(54) CometSort Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST], false, 0 +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] -(63) SortMergeJoin [codegen id : 21] -Left keys [2]: [ws_order_number#41, ws_item_sk#40] -Right keys [2]: [wr_order_number#53, wr_item_sk#52] -Join type: LeftOuter -Join condition: None +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter -(64) Project [codegen id : 21] -Output [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] +(56) CometProject Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] -(65) Union +(57) CometUnion +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] -(66) HashAggregate [codegen id : 22] +(58) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(67) Exchange +(59) CometExchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(68) HashAggregate [codegen id : 23] +(60) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(69) HashAggregate [codegen id : 23] +(61) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#60] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -(70) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(71) HashAggregate [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] +(63) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#63 AS sales_cnt#65, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#66] -(72) Filter [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Condition : isnotnull(sales_cnt#65) +(64) CometFilter +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) -(73) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(65) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(74) Sort [codegen id : 25] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 +(66) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] -(75) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(67) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#71)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(68) CometFilter +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) -(77) Filter [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Condition : isnotnull(cs_item_sk#67) +(69) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(78) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(70) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight -(79) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_item_sk#67] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(71) CometProject +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(80) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Input [10]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +(72) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(82) ColumnarToRow [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] - -(83) Filter [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +(73) CometFilter +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) -(84) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(74) CometBroadcastExchange +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] -(85) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_sold_date_sk#71] -Right keys [1]: [d_date_sk#77] -Join type: Inner -Join condition: None +(75) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight -(86) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Input [11]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] +(76) CometProject +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] -(87) Exchange -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#68, cs_item_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(77) CometExchange +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(88) Sort [codegen id : 29] -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_order_number#68 ASC NULLS FIRST, cs_item_sk#67 ASC NULLS FIRST], false, 0 +(78) CometSort +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] -(89) ReusedExchange [Reuses operator id: 23] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(79) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] -(90) Sort [codegen id : 31] -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST], false, 0 +(80) CometSort +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] -(91) SortMergeJoin [codegen id : 32] -Left keys [2]: [cs_order_number#68, cs_item_sk#67] -Right keys [2]: [cr_order_number#80, cr_item_sk#79] -Join type: LeftOuter -Join condition: None +(81) CometSortMergeJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter -(92) Project [codegen id : 32] -Output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#69 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#19, (cs_ext_sales_price#70 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(82) CometProject +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] -(93) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(83) CometNativeScan parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(94) ColumnarToRow [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(84) CometFilter +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) -(95) Filter [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +(85) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(96) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(86) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight -(97) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_item_sk#83] -Right keys [1]: [i_item_sk#88] -Join type: Inner -Join condition: None +(87) CometProject +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(98) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(88) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#89, d_year#90] -(99) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#93, d_year#94] +(89) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_sold_date_sk#87] -Right keys [1]: [d_date_sk#93] -Join type: Inner -Join condition: None +(90) CometProject +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] -(101) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +(91) CometExchange +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(102) Exchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=14] +(92) CometSort +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] -(103) Sort [codegen id : 36] -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST], false, 0 +(93) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] -(104) ReusedExchange [Reuses operator id: 42] -Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +(94) CometSort +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] -(105) Sort [codegen id : 38] -Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] -Arguments: [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST], false, 0 +(95) CometSortMergeJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter -(106) SortMergeJoin [codegen id : 39] -Left keys [2]: [ss_ticket_number#84, ss_item_sk#83] -Right keys [2]: [sr_ticket_number#96, sr_item_sk#95] -Join type: LeftOuter -Join condition: None +(96) CometProject +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] -(107) Project [codegen id : 39] -Output [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#38, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(108) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] +(97) CometNativeScan parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#103)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(109) ColumnarToRow [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] - -(110) Filter [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Condition : isnotnull(ws_item_sk#99) +(98) CometFilter +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) -(111) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(99) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(112) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_item_sk#99] -Right keys [1]: [i_item_sk#104] -Join type: Inner -Join condition: None +(100) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight -(113) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] -Input [10]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(101) CometProject +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(114) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#109, d_year#110] +(102) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#105, d_year#106] -(115) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_sold_date_sk#103] -Right keys [1]: [d_date_sk#109] -Join type: Inner -Join condition: None +(103) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight -(116) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Input [11]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_date_sk#109, d_year#110] +(104) CometProject +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] -(117) Exchange -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: hashpartitioning(ws_order_number#100, ws_item_sk#99, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(105) CometExchange +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(118) Sort [codegen id : 43] -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: [ws_order_number#100 ASC NULLS FIRST, ws_item_sk#99 ASC NULLS FIRST], false, 0 +(106) CometSort +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] -(119) ReusedExchange [Reuses operator id: 61] -Output [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(107) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] -(120) Sort [codegen id : 45] -Input [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] -Arguments: [wr_order_number#112 ASC NULLS FIRST, wr_item_sk#111 ASC NULLS FIRST], false, 0 +(108) CometSort +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] -(121) SortMergeJoin [codegen id : 46] -Left keys [2]: [ws_order_number#100, ws_item_sk#99] -Right keys [2]: [wr_order_number#112, wr_item_sk#111] -Join type: LeftOuter -Join condition: None +(109) CometSortMergeJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter -(122) Project [codegen id : 46] -Output [7]: [d_year#110, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, (ws_quantity#101 - coalesce(wr_return_quantity#113, 0)) AS sales_cnt#57, (ws_ext_sales_price#102 - coalesce(wr_return_amt#114, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110, wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(110) CometProject +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] -(123) Union +(111) CometUnion +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] -(124) HashAggregate [codegen id : 47] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(112) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(125) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(113) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(126) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(114) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(127) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(115) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#115] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -(128) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=17] +(116) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(129) HashAggregate [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(117) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum(sales_cnt#19)#63 AS sales_cnt#117, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#118] - -(130) Filter [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Condition : isnotnull(sales_cnt#117) - -(131) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(132) Sort [codegen id : 50] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST], false, 0 - -(133) SortMergeJoin [codegen id : 51] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Join type: Inner -Join condition: ((cast(sales_cnt#65 as decimal(17,2)) / cast(sales_cnt#117 as decimal(17,2))) < 0.90000000000000000000) - -(134) Project [codegen id : 51] -Output [10]: [d_year#78 AS prev_year#119, d_year#13 AS year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#117 AS prev_yr_cnt#121, sales_cnt#65 AS curr_yr_cnt#122, (sales_cnt#65 - sales_cnt#117) AS sales_cnt_diff#123, (sales_amt#66 - sales_amt#118) AS sales_amt_diff#124] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] - -(135) TakeOrderedAndProject -Input [10]: [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] -Arguments: 100, [sales_cnt_diff#123 ASC NULLS FIRST, sales_amt_diff#124 ASC NULLS FIRST], [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] + +(118) CometFilter +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) + +(119) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] + +(120) CometSort +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] + +(121) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) + +(122) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] + +(123) CometTakeOrderedAndProject +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST,sales_amt_diff#119 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST, sales_amt_diff#119 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] + +(124) CometColumnarToRow [codegen id : 1] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/extended.txt new file mode 100644 index 0000000000..84620b41ae --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/extended.txt @@ -0,0 +1,158 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : :- CometProject + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.item + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_returns + : +- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_returns + +- CometSort + +- CometExchange + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store_returns + +- CometProject + +- CometSortMergeJoin + :- CometSort + : +- CometExchange + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_returns + +Comet accelerated 155 out of 155 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/simplified.txt index 8fd51946e0..588e8943c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/simplified.txt @@ -1,232 +1,126 @@ -TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt] - WholeStageCodegen (51) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (25) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (24) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (23) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - WholeStageCodegen (22) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (7) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #4 - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (6) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #7 - WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - WholeStageCodegen (14) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (11) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #8 - WholeStageCodegen (10) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (13) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #9 - WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - WholeStageCodegen (21) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #10 - WholeStageCodegen (17) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (20) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #11 - WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (50) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 - WholeStageCodegen (49) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - WholeStageCodegen (48) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 - WholeStageCodegen (47) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (32) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #15 - WholeStageCodegen (28) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (27) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (31) - Sort [cr_order_number,cr_item_sk] - InputAdapter - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - WholeStageCodegen (39) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (36) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 - WholeStageCodegen (35) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (38) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 - WholeStageCodegen (46) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (43) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 - WholeStageCodegen (42) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (45) - Sort [wr_order_number,wr_item_sk] - InputAdapter - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/explain.txt index 8f0ae3b3f2..f3c8ad9b6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/explain.txt @@ -1,612 +1,578 @@ == Physical Plan == -TakeOrderedAndProject (102) -+- * HashAggregate (101) - +- Exchange (100) - +- * HashAggregate (99) - +- Union (98) - :- * HashAggregate (87) - : +- Exchange (86) - : +- * HashAggregate (85) - : +- Union (84) - : :- * Project (34) - : : +- * BroadcastHashJoin LeftOuter BuildRight (33) - : : :- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.store (11) - : : +- BroadcastExchange (32) - : : +- * HashAggregate (31) - : : +- Exchange (30) - : : +- * HashAggregate (29) - : : +- * Project (28) - : : +- * BroadcastHashJoin Inner BuildRight (27) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.store_returns (20) - : : : +- ReusedExchange (23) - : : +- ReusedExchange (26) - : :- * Project (53) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (52) - : : :- BroadcastExchange (43) - : : : +- * HashAggregate (42) - : : : +- Exchange (41) - : : : +- * HashAggregate (40) - : : : +- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * ColumnarToRow (36) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (35) - : : : +- ReusedExchange (37) - : : +- * HashAggregate (51) - : : +- Exchange (50) - : : +- * HashAggregate (49) - : : +- * Project (48) - : : +- * BroadcastHashJoin Inner BuildRight (47) - : : :- * ColumnarToRow (45) - : : : +- Scan parquet spark_catalog.default.catalog_returns (44) - : : +- ReusedExchange (46) - : +- * Project (83) - : +- * BroadcastHashJoin LeftOuter BuildRight (82) - : :- * HashAggregate (68) - : : +- Exchange (67) - : : +- * HashAggregate (66) - : : +- * Project (65) - : : +- * BroadcastHashJoin Inner BuildRight (64) - : : :- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : :- * Filter (56) - : : : : +- * ColumnarToRow (55) - : : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : : +- ReusedExchange (57) - : : +- BroadcastExchange (63) - : : +- * Filter (62) - : : +- * ColumnarToRow (61) - : : +- Scan parquet spark_catalog.default.web_page (60) - : +- BroadcastExchange (81) - : +- * HashAggregate (80) - : +- Exchange (79) - : +- * HashAggregate (78) - : +- * Project (77) - : +- * BroadcastHashJoin Inner BuildRight (76) - : :- * Project (74) - : : +- * BroadcastHashJoin Inner BuildRight (73) - : : :- * Filter (71) - : : : +- * ColumnarToRow (70) - : : : +- Scan parquet spark_catalog.default.web_returns (69) - : : +- ReusedExchange (72) - : +- ReusedExchange (75) - :- * HashAggregate (92) - : +- Exchange (91) - : +- * HashAggregate (90) - : +- * HashAggregate (89) - : +- ReusedExchange (88) - +- * HashAggregate (97) - +- Exchange (96) - +- * HashAggregate (95) - +- * HashAggregate (94) - +- ReusedExchange (93) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (103) ++- CometTakeOrderedAndProject (102) + +- CometHashAggregate (101) + +- CometColumnarExchange (100) + +- * HashAggregate (99) + +- Union (98) + :- * HashAggregate (83) + : +- * CometColumnarToRow (82) + : +- CometColumnarExchange (81) + : +- * HashAggregate (80) + : +- Union (79) + : :- * CometColumnarToRow (31) + : : +- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometNativeScan parquet spark_catalog.default.store (9) + : : +- CometBroadcastExchange (28) + : : +- CometHashAggregate (27) + : : +- CometExchange (26) + : : +- CometHashAggregate (25) + : : +- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (21) + : : : +- CometBroadcastHashJoin (20) + : : : :- CometFilter (18) + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (17) + : : : +- ReusedExchange (19) + : : +- ReusedExchange (22) + : :- * Project (50) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : : :- BroadcastExchange (40) + : : : +- * CometColumnarToRow (39) + : : : +- CometHashAggregate (38) + : : : +- CometExchange (37) + : : : +- CometHashAggregate (36) + : : : +- CometProject (35) + : : : +- CometBroadcastHashJoin (34) + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales (32) + : : : +- ReusedExchange (33) + : : +- * CometColumnarToRow (48) + : : +- CometHashAggregate (47) + : : +- CometExchange (46) + : : +- CometHashAggregate (45) + : : +- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns (41) + : : +- ReusedExchange (42) + : +- * CometColumnarToRow (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometFilter (52) + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (51) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometNativeScan parquet spark_catalog.default.web_page (56) + : +- CometBroadcastExchange (75) + : +- CometHashAggregate (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometNativeScan parquet spark_catalog.default.web_returns (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + :- * HashAggregate (90) + : +- * CometColumnarToRow (89) + : +- CometColumnarExchange (88) + : +- * HashAggregate (87) + : +- * HashAggregate (86) + : +- * CometColumnarToRow (85) + : +- ReusedExchange (84) + +- * HashAggregate (97) + +- * CometColumnarToRow (96) + +- CometColumnarExchange (95) + +- * HashAggregate (94) + +- * HashAggregate (93) + +- * CometColumnarToRow (92) + +- ReusedExchange (91) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_date#6] Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +(8) CometProject Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -(11) Scan parquet spark_catalog.default.store +(9) CometNativeScan parquet spark_catalog.default.store Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [s_store_sk#7] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [1]: [s_store_sk#7] Condition : isnotnull(s_store_sk#7) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#7] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +(13) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -(17) HashAggregate [codegen id : 3] +(14) CometHashAggregate Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] -(18) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 8] -Input [3]: [s_store_sk#7, sum#10, sum#11] +(16) CometHashAggregate +Input [3]: [s_store_sk#7, sum#8, sum#9] Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] -(20) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(17) CometNativeScan parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#19)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(18) CometFilter +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) -(22) Filter [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) +(19) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#14] -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] +(20) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_returned_date_sk#19] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None +(21) CometProject +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] -(25) Project [codegen id : 6] -Output [3]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18] -Input [5]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19, d_date_sk#20] +(22) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#15] -(26) ReusedExchange [Reuses operator id: 14] -Output [1]: [s_store_sk#21] +(23) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_store_sk#16] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None +(24) CometProject +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] -(28) Project [codegen id : 6] -Output [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, s_store_sk#21] - -(29) HashAggregate [codegen id : 6] -Input [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#17)), partial_sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum#22, sum#23] -Results [3]: [s_store_sk#21, sum#24, sum#25] - -(30) Exchange -Input [3]: [s_store_sk#21, sum#24, sum#25] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [s_store_sk#21, sum#24, sum#25] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#17)), sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#17))#26, sum(UnscaledValue(sr_net_loss#18))#27] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#17))#26,17,2) AS returns#28, MakeDecimal(sum(UnscaledValue(sr_net_loss#18))#27,17,2) AS profit_loss#29] - -(32) BroadcastExchange -Input [3]: [s_store_sk#21, returns#28, profit_loss#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#21] -Join type: LeftOuter -Join condition: None +(25) CometHashAggregate +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] -(34) Project [codegen id : 8] -Output [5]: [store channel AS channel#30, s_store_sk#7 AS id#31, sales#14, coalesce(returns#28, 0.00) AS returns#32, (profit#15 - coalesce(profit_loss#29, 0.00)) AS profit#33] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#21, returns#28, profit_loss#29] +(26) CometExchange +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(27) CometHashAggregate +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] + +(28) CometBroadcastExchange +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] + +(29) CometBroadcastHashJoin +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight + +(30) CometProject +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [channel#22, id#23, sales#20, returns#24, profit#25], [store channel AS channel#22, s_store_sk#7 AS id#23, sales#20, coalesce(returns#18, 0.00) AS returns#24, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#25] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] + +(32) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#37)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(33) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] -(37) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#38] +(34) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None +(35) CometProject +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] -(39) Project [codegen id : 10] -Output [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Input [5]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37, d_date_sk#38] - -(40) HashAggregate [codegen id : 10] -Input [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#35)), partial_sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum#39, sum#40] -Results [3]: [cs_call_center_sk#34, sum#41, sum#42] - -(41) Exchange -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Arguments: hashpartitioning(cs_call_center_sk#34, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#35)), sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#35))#43, sum(UnscaledValue(cs_net_profit#36))#44] -Results [3]: [cs_call_center_sk#34, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#35))#43,17,2) AS sales#45, MakeDecimal(sum(UnscaledValue(cs_net_profit#36))#44,17,2) AS profit#46] - -(43) BroadcastExchange -Input [3]: [cs_call_center_sk#34, sales#45, profit#46] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(44) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] +(36) CometHashAggregate +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] + +(37) CometExchange +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometHashAggregate +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] + +(39) CometColumnarToRow [codegen id : 2] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] + +(40) BroadcastExchange +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] +Arguments: IdentityBroadcastMode, [plan_id=4] + +(41) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#49)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 13] -Input [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] - -(46) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#50] +(42) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#38] -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [cr_returned_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight -(48) Project [codegen id : 13] -Output [2]: [cr_return_amount#47, cr_net_loss#48] -Input [4]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49, d_date_sk#50] +(44) CometProject +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] -(49) HashAggregate [codegen id : 13] -Input [2]: [cr_return_amount#47, cr_net_loss#48] +(45) CometHashAggregate +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#47)), partial_sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [2]: [sum#53, sum#54] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] -(50) Exchange -Input [2]: [sum#53, sum#54] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [2]: [sum#39, sum#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(51) HashAggregate -Input [2]: [sum#53, sum#54] +(47) CometHashAggregate +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#47)), sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#47))#55, sum(UnscaledValue(cr_net_loss#48))#56] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#47))#55,17,2) AS returns#57, MakeDecimal(sum(UnscaledValue(cr_net_loss#48))#56,17,2) AS profit_loss#58] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] + +(48) CometColumnarToRow +Input [2]: [returns#41, profit_loss#42] -(52) BroadcastNestedLoopJoin [codegen id : 14] +(49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None -(53) Project [codegen id : 14] -Output [5]: [catalog channel AS channel#59, cs_call_center_sk#34 AS id#60, sales#45, returns#57, (profit#46 - profit_loss#58) AS profit#61] -Input [5]: [cs_call_center_sk#34, sales#45, profit#46, returns#57, profit_loss#58] +(50) Project [codegen id : 3] +Output [5]: [catalog channel AS channel#43, cs_call_center_sk#26 AS id#44, sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] +(51) CometNativeScan parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#65)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] +(52) CometFilter +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) -(56) Filter [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Condition : isnotnull(ws_web_page_sk#62) - -(57) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#66] +(53) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#50] -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None +(54) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight -(59) Project [codegen id : 17] -Output [3]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64] -Input [5]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65, d_date_sk#66] +(55) CometProject +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] -(60) Scan parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#67] +(56) CometNativeScan parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [1]: [wp_web_page_sk#67] +(57) CometFilter +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) -(62) Filter [codegen id : 16] -Input [1]: [wp_web_page_sk#67] -Condition : isnotnull(wp_web_page_sk#67) +(58) CometBroadcastExchange +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] -(63) BroadcastExchange -Input [1]: [wp_web_page_sk#67] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(59) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight -(64) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_web_page_sk#62] -Right keys [1]: [wp_web_page_sk#67] -Join type: Inner -Join condition: None +(60) CometProject +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] -(65) Project [codegen id : 17] -Output [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] - -(66) HashAggregate [codegen id : 17] -Input [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#63)), partial_sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum#68, sum#69] -Results [3]: [wp_web_page_sk#67, sum#70, sum#71] - -(67) Exchange -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Arguments: hashpartitioning(wp_web_page_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#63)), sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#63))#72, sum(UnscaledValue(ws_net_profit#64))#73] -Results [3]: [wp_web_page_sk#67, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#63))#72,17,2) AS sales#74, MakeDecimal(sum(UnscaledValue(ws_net_profit#64))#73,17,2) AS profit#75] - -(69) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#79)] -PushedFilters: [IsNotNull(wr_web_page_sk)] -ReadSchema: struct - -(70) ColumnarToRow [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] +(61) CometHashAggregate +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] -(71) Filter [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Condition : isnotnull(wr_web_page_sk#76) +(62) CometExchange +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(72) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#80] +(63) CometHashAggregate +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] -(73) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_returned_date_sk#79] -Right keys [1]: [d_date_sk#80] -Join type: Inner -Join condition: None - -(74) Project [codegen id : 20] -Output [3]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78] -Input [5]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79, d_date_sk#80] - -(75) ReusedExchange [Reuses operator id: 63] -Output [1]: [wp_web_page_sk#81] - -(76) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_web_page_sk#76] -Right keys [1]: [wp_web_page_sk#81] -Join type: Inner -Join condition: None - -(77) Project [codegen id : 20] -Output [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] - -(78) HashAggregate [codegen id : 20] -Input [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#77)), partial_sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum#82, sum#83] -Results [3]: [wp_web_page_sk#81, sum#84, sum#85] - -(79) Exchange -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Arguments: hashpartitioning(wp_web_page_sk#81, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 21] -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [sum(UnscaledValue(wr_return_amt#77)), sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#77))#86, sum(UnscaledValue(wr_net_loss#78))#87] -Results [3]: [wp_web_page_sk#81, MakeDecimal(sum(UnscaledValue(wr_return_amt#77))#86,17,2) AS returns#88, MakeDecimal(sum(UnscaledValue(wr_net_loss#78))#87,17,2) AS profit_loss#89] - -(81) BroadcastExchange -Input [3]: [wp_web_page_sk#81, returns#88, profit_loss#89] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(82) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [wp_web_page_sk#67] -Right keys [1]: [wp_web_page_sk#81] -Join type: LeftOuter -Join condition: None +(64) CometNativeScan parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Batched: true +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct -(83) Project [codegen id : 22] -Output [5]: [web channel AS channel#90, wp_web_page_sk#67 AS id#91, sales#74, coalesce(returns#88, 0.00) AS returns#92, (profit#75 - coalesce(profit_loss#89, 0.00)) AS profit#93] -Input [6]: [wp_web_page_sk#67, sales#74, profit#75, wp_web_page_sk#81, returns#88, profit_loss#89] - -(84) Union - -(85) HashAggregate [codegen id : 23] -Input [5]: [channel#30, id#31, sales#14, returns#32, profit#33] -Keys [2]: [channel#30, id#31] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Results [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(86) Exchange -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(87) HashAggregate [codegen id : 24] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [5]: [channel#30, id#31, cast(sum(sales#14)#106 as decimal(37,2)) AS sales#109, cast(sum(returns#32)#107 as decimal(37,2)) AS returns#110, cast(sum(profit#33)#108 as decimal(38,2)) AS profit#111] - -(88) ReusedExchange [Reuses operator id: 86] -Output [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(89) HashAggregate [codegen id : 48] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [4]: [channel#30, sum(sales#14)#106 AS sales#112, sum(returns#32)#107 AS returns#113, sum(profit#33)#108 AS profit#114] - -(90) HashAggregate [codegen id : 48] -Input [4]: [channel#30, sales#112, returns#113, profit#114] -Keys [1]: [channel#30] -Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -Aggregate Attributes [6]: [sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Results [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(91) Exchange -Input [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Arguments: hashpartitioning(channel#30, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(92) HashAggregate [codegen id : 49] -Input [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Keys [1]: [channel#30] -Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -Aggregate Attributes [3]: [sum(sales#112)#127, sum(returns#113)#128, sum(profit#114)#129] -Results [5]: [channel#30, null AS id#130, sum(sales#112)#127 AS sales#131, sum(returns#113)#128 AS returns#132, sum(profit#114)#129 AS profit#133] - -(93) ReusedExchange [Reuses operator id: 86] -Output [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(94) HashAggregate [codegen id : 73] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [3]: [sum(sales#14)#106 AS sales#112, sum(returns#32)#107 AS returns#113, sum(profit#33)#108 AS profit#114] - -(95) HashAggregate [codegen id : 73] -Input [3]: [sales#112, returns#113, profit#114] +(65) CometFilter +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) + +(66) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(67) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(68) CometProject +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] + +(69) ReusedExchange [Reuses operator id: 58] +Output [1]: [wp_web_page_sk#59] + +(70) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(71) CometProject +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] + +(72) CometHashAggregate +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] + +(73) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(74) CometHashAggregate +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] + +(75) CometBroadcastExchange +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] + +(76) CometBroadcastHashJoin +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight + +(77) CometProject +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [channel#66, id#67, sales#64, returns#68, profit#69], [web channel AS channel#66, wp_web_page_sk#51 AS id#67, sales#64, coalesce(returns#62, 0.00) AS returns#68, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#69] + +(78) CometColumnarToRow [codegen id : 4] +Input [5]: [channel#66, id#67, sales#64, returns#68, profit#69] + +(79) Union + +(80) HashAggregate [codegen id : 5] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] +Keys [2]: [channel#22, id#23] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#24), partial_sum(profit#25)] +Aggregate Attributes [6]: [sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75] +Results [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(81) CometColumnarExchange +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Arguments: hashpartitioning(channel#22, id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(82) CometColumnarToRow [codegen id : 6] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(83) HashAggregate [codegen id : 6] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [5]: [channel#22, id#23, cast(sum(sales#20)#82 as decimal(37,2)) AS sales#85, cast(sum(returns#24)#83 as decimal(37,2)) AS returns#86, cast(sum(profit#25)#84 as decimal(38,2)) AS profit#87] + +(84) ReusedExchange [Reuses operator id: 81] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(85) CometColumnarToRow [codegen id : 12] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(86) HashAggregate [codegen id : 12] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [4]: [channel#22, sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] + +(87) HashAggregate [codegen id : 12] +Input [4]: [channel#22, sales#88, returns#89, profit#90] +Keys [1]: [channel#22] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Results [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] + +(88) CometColumnarExchange +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Arguments: hashpartitioning(channel#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(89) CometColumnarToRow [codegen id : 13] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] + +(90) HashAggregate [codegen id : 13] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Keys [1]: [channel#22] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#103, sum(returns#89)#104, sum(profit#90)#105] +Results [5]: [channel#22, null AS id#106, sum(sales#88)#103 AS sales#107, sum(returns#89)#104 AS returns#108, sum(profit#90)#105 AS profit#109] + +(91) ReusedExchange [Reuses operator id: 81] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(92) CometColumnarToRow [codegen id : 19] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(93) HashAggregate [codegen id : 19] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [3]: [sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] + +(94) HashAggregate [codegen id : 19] +Input [3]: [sales#88, returns#89, profit#90] Keys: [] -Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -Aggregate Attributes [6]: [sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Results [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115] +Results [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] -(96) Exchange -Input [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(95) CometColumnarExchange +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(97) HashAggregate [codegen id : 74] -Input [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] +(96) CometColumnarToRow [codegen id : 20] +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] + +(97) HashAggregate [codegen id : 20] +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] Keys: [] -Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -Aggregate Attributes [3]: [sum(sales#112)#146, sum(returns#113)#147, sum(profit#114)#148] -Results [5]: [null AS channel#149, null AS id#150, sum(sales#112)#146 AS sales#151, sum(returns#113)#147 AS returns#152, sum(profit#114)#148 AS profit#153] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#122, sum(returns#89)#123, sum(profit#90)#124] +Results [5]: [null AS channel#125, null AS id#126, sum(sales#88)#122 AS sales#127, sum(returns#89)#123 AS returns#128, sum(profit#90)#124 AS profit#129] (98) Union -(99) HashAggregate [codegen id : 75] -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Keys [5]: [channel#30, id#31, sales#109, returns#110, profit#111] +(99) HashAggregate [codegen id : 21] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#30, id#31, sales#109, returns#110, profit#111] +Results [5]: [channel#22, id#23, sales#85, returns#86, profit#87] -(100) Exchange -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Arguments: hashpartitioning(channel#30, id#31, sales#109, returns#110, profit#111, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(100) CometColumnarExchange +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: hashpartitioning(channel#22, id#23, sales#85, returns#86, profit#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] -(101) HashAggregate [codegen id : 76] -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Keys [5]: [channel#30, id#31, sales#109, returns#110, profit#111] +(101) CometHashAggregate +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -(102) TakeOrderedAndProject -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Arguments: 100, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#109, returns#110, profit#111] +(102) CometTakeOrderedAndProject +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#22 ASC NULLS FIRST,id#23 ASC NULLS FIRST], output=[channel#22,id#23,sales#85,returns#86,profit#87]), [channel#22, id#23, sales#85, returns#86, profit#87], 100, 0, [channel#22 ASC NULLS FIRST, id#23 ASC NULLS FIRST], [channel#22, id#23, sales#85, returns#86, profit#87] + +(103) CometColumnarToRow [codegen id : 22] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/extended.txt new file mode 100644 index 0000000000..13d4b5c286 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/extended.txt @@ -0,0 +1,322 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometColumnarExchange + +- HashAggregate + +- Union + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + :- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- HashAggregate + : +- CometColumnarToRow + : +- CometColumnarExchange + : +- HashAggregate + : +- Union + : :- CometColumnarToRow + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : :- Project + : : +- BroadcastNestedLoopJoin + : : :- BroadcastExchange + : : : +- CometColumnarToRow + : : : +- CometHashAggregate + : : : +- CometExchange + : : : +- CometHashAggregate + : : : +- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_page + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- HashAggregate + +- CometColumnarToRow + +- CometColumnarExchange + +- HashAggregate + +- Union + :- CometColumnarToRow + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- CometBroadcastExchange + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.store + :- Project + : +- BroadcastNestedLoopJoin + : :- BroadcastExchange + : : +- CometColumnarToRow + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometNativeScan parquet spark_catalog.default.catalog_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometColumnarToRow + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometNativeScan parquet spark_catalog.default.catalog_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometColumnarToRow + +- CometProject + +- CometBroadcastHashJoin + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.web_page + +- CometBroadcastExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.web_page + +Comet accelerated 278 out of 302 eligible operators (92%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/simplified.txt index 565a880398..cc8a78ab5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/simplified.txt @@ -1,161 +1,128 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (76) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (75) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (24) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (23) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (8) - Project [s_store_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] +WholeStageCodegen (22) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (21) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (6) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (5) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [s_store_sk] #3 - WholeStageCodegen (3) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,s_store_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [s_store_sk] #7 - WholeStageCodegen (6) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - Project [sr_return_amt,sr_net_loss,s_store_sk] - BroadcastHashJoin [sr_store_sk,s_store_sk] - Project [sr_store_sk,sr_return_amt,sr_net_loss] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [s_store_sk] #5 - WholeStageCodegen (14) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (11) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #9 - WholeStageCodegen (10) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange #10 - WholeStageCodegen (13) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - Project [cr_return_amount,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (22) - Project [wp_web_page_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [s_store_sk,sales,profit,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #3 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometNativeScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 + CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #7 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (3) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #9 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #10 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Exchange [wp_web_page_sk] #11 - WholeStageCodegen (17) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (21) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #14 - WholeStageCodegen (20) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - Project [wr_return_amt,wr_net_loss,wp_web_page_sk] - BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] - Project [wr_web_page_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [wp_web_page_sk] #12 - WholeStageCodegen (49) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #15 - WholeStageCodegen (48) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (74) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #16 - WholeStageCodegen (73) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #11 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometNativeScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 + CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #14 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 + WholeStageCodegen (13) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #15 + WholeStageCodegen (12) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (20) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #16 + WholeStageCodegen (19) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/explain.txt index 57fc823955..e26c10beab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/explain.txt @@ -1,417 +1,373 @@ == Physical Plan == -TakeOrderedAndProject (73) -+- * Project (72) - +- * SortMergeJoin Inner (71) - :- * Project (48) - : +- * SortMergeJoin Inner (47) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Project (14) - : : : +- * Filter (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- * Sort (46) - : +- * Filter (45) - : +- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * Project (41) - : +- * BroadcastHashJoin Inner BuildRight (40) - : :- * Project (38) - : : +- * Filter (37) - : : +- * SortMergeJoin LeftOuter (36) - : : :- * Sort (29) - : : : +- Exchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.web_sales (25) - : : +- * Sort (35) - : : +- Exchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.web_returns (30) - : +- ReusedExchange (39) - +- * Sort (70) - +- * Filter (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * Filter (61) - : +- * SortMergeJoin LeftOuter (60) - : :- * Sort (53) - : : +- Exchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.catalog_sales (49) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- Scan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (67) ++- * Project (66) + +- * CometColumnarToRow (65) + +- CometSortMergeJoin (64) + :- CometProject (43) + : +- CometSortMergeJoin (42) + : :- CometSort (21) + : : +- CometHashAggregate (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (13) + : +- CometSort (41) + : +- CometFilter (40) + : +- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (33) + : : +- CometFilter (32) + : : +- CometSortMergeJoin (31) + : : :- CometSort (25) + : : : +- CometExchange (24) + : : : +- CometFilter (23) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (22) + : : +- CometSort (30) + : : +- CometExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometNativeScan parquet spark_catalog.default.web_returns (26) + : +- ReusedExchange (34) + +- CometSort (63) + +- CometFilter (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometFilter (54) + : +- CometSortMergeJoin (53) + : :- CometSort (47) + : : +- CometExchange (46) + : : +- CometFilter (45) + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (44) + : +- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometFilter (49) + : +- CometNativeScan parquet spark_catalog.default.catalog_returns (48) + +- ReusedExchange (56) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#8, sr_ticket_number#9] +(7) CometProject Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] -(10) Exchange +(8) CometExchange Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#9, sr_item_sk#8] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter -(13) Filter [codegen id : 6] +(11) CometFilter Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Condition : isnull(sr_ticket_number#9) -(14) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +(12) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(15) Scan parquet spark_catalog.default.date_dim +(13) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] - -(17) Filter [codegen id : 5] +(14) CometFilter Input [2]: [d_date_sk#11, d_year#12] Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) -(18) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [d_date_sk#11, d_year#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#11, d_year#12] -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight -(20) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +(17) CometProject Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -(22) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) HashAggregate [codegen id : 7] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] +(20) CometHashAggregate +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] -(24) Sort [codegen id : 7] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 +(21) CometSort +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(25) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(22) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#32)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(23) CometFilter +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) -(27) Filter [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Condition : (isnotnull(ws_item_sk#26) AND isnotnull(ws_bill_customer_sk#27)) +(24) CometExchange +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(28) Exchange -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: hashpartitioning(ws_order_number#28, ws_item_sk#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometSort +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(29) Sort [codegen id : 9] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: [ws_order_number#28 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] +(26) CometNativeScan parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(32) Filter [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Condition : (isnotnull(wr_order_number#34) AND isnotnull(wr_item_sk#33)) - -(33) Project [codegen id : 10] -Output [2]: [wr_item_sk#33, wr_order_number#34] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(34) Exchange -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: hashpartitioning(wr_order_number#34, wr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 11] -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: [wr_order_number#34 ASC NULLS FIRST, wr_item_sk#33 ASC NULLS FIRST], false, 0 - -(36) SortMergeJoin [codegen id : 13] -Left keys [2]: [ws_order_number#28, ws_item_sk#26] -Right keys [2]: [wr_order_number#34, wr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(37) Filter [codegen id : 13] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] -Condition : isnull(wr_order_number#34) - -(38) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] - -(39) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#36, d_year#37] - -(40) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#32] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Input [8]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, d_date_sk#36, d_year#37] - -(42) HashAggregate [codegen id : 13] -Input [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [partial_sum(ws_quantity#29), partial_sum(UnscaledValue(ws_wholesale_cost#30)), partial_sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum#38, sum#39, sum#40] -Results [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] - -(43) Exchange -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Arguments: hashpartitioning(d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(44) HashAggregate [codegen id : 14] -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [sum(ws_quantity#29), sum(UnscaledValue(ws_wholesale_cost#30)), sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum(ws_quantity#29)#44, sum(UnscaledValue(ws_wholesale_cost#30))#45, sum(UnscaledValue(ws_sales_price#31))#46] -Results [6]: [d_year#37 AS ws_sold_year#47, ws_item_sk#26, ws_bill_customer_sk#27 AS ws_customer_sk#48, sum(ws_quantity#29)#44 AS ws_qty#49, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#30))#45,17,2) AS ws_wc#50, MakeDecimal(sum(UnscaledValue(ws_sales_price#31))#46,17,2) AS ws_sp#51] - -(45) Filter [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Condition : (coalesce(ws_qty#49, 0) > 0) - -(46) Sort [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Arguments: [ws_sold_year#47 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST, ws_customer_sk#48 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 15] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 15] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] - -(49) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] +(27) CometFilter +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) + +(28) CometProject +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] + +(29) CometExchange +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(30) CometSort +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] + +(31) CometSortMergeJoin +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter + +(32) CometFilter +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) + +(33) CometProject +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] + +(34) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#30, d_year#31] + +(35) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight + +(36) CometProject +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] + +(37) CometHashAggregate +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] + +(38) CometExchange +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometHashAggregate +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] + +(40) CometFilter +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) + +(41) CometSort +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] + +(42) CometSortMergeJoin +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner + +(43) CometProject +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] + +(44) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] - -(51) Filter [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Condition : (isnotnull(cs_item_sk#53) AND isnotnull(cs_bill_customer_sk#52)) +(45) CometFilter +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) -(52) Exchange -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: hashpartitioning(cs_order_number#54, cs_item_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(53) Sort [codegen id : 17] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: [cs_order_number#54 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST], false, 0 +(47) CometSort +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] -(54) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] +(48) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(56) Filter [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Condition : (isnotnull(cr_order_number#60) AND isnotnull(cr_item_sk#59)) - -(57) Project [codegen id : 18] -Output [2]: [cr_item_sk#59, cr_order_number#60] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(58) Exchange -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: hashpartitioning(cr_order_number#60, cr_item_sk#59, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(59) Sort [codegen id : 19] -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: [cr_order_number#60 ASC NULLS FIRST, cr_item_sk#59 ASC NULLS FIRST], false, 0 - -(60) SortMergeJoin [codegen id : 21] -Left keys [2]: [cs_order_number#54, cs_item_sk#53] -Right keys [2]: [cr_order_number#60, cr_item_sk#59] -Join type: LeftOuter -Join condition: None - -(61) Filter [codegen id : 21] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] -Condition : isnull(cr_order_number#60) - -(62) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] - -(63) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#62, d_year#63] - -(64) BroadcastHashJoin [codegen id : 21] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Input [8]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, d_date_sk#62, d_year#63] - -(66) HashAggregate [codegen id : 21] -Input [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [partial_sum(cs_quantity#55), partial_sum(UnscaledValue(cs_wholesale_cost#56)), partial_sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum#64, sum#65, sum#66] -Results [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] - -(67) Exchange -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Arguments: hashpartitioning(d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [sum(cs_quantity#55), sum(UnscaledValue(cs_wholesale_cost#56)), sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum(cs_quantity#55)#70, sum(UnscaledValue(cs_wholesale_cost#56))#71, sum(UnscaledValue(cs_sales_price#57))#72] -Results [6]: [d_year#63 AS cs_sold_year#73, cs_item_sk#53, cs_bill_customer_sk#52 AS cs_customer_sk#74, sum(cs_quantity#55)#70 AS cs_qty#75, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#56))#71,17,2) AS cs_wc#76, MakeDecimal(sum(UnscaledValue(cs_sales_price#57))#72,17,2) AS cs_sp#77] - -(69) Filter [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Condition : (coalesce(cs_qty#75, 0) > 0) - -(70) Sort [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Arguments: [cs_sold_year#73 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST, cs_customer_sk#74 ASC NULLS FIRST], false, 0 - -(71) SortMergeJoin [codegen id : 23] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74] -Join type: Inner -Join condition: None - -(72) Project [codegen id : 23] -Output [13]: [round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) AS ratio#78, ss_qty#23 AS store_qty#79, ss_wc#24 AS store_wholesale_cost#80, ss_sp#25 AS store_sales_price#81, (coalesce(ws_qty#49, 0) + coalesce(cs_qty#75, 0)) AS other_chan_qty#82, (coalesce(ws_wc#50, 0.00) + coalesce(cs_wc#76, 0.00)) AS other_chan_wholesale_cost#83, (coalesce(ws_sp#51, 0.00) + coalesce(cs_sp#77, 0.00)) AS other_chan_sales_price#84, ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51, cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] - -(73) TakeOrderedAndProject -Input [13]: [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84, ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: 100, [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#82 ASC NULLS FIRST, other_chan_wholesale_cost#83 ASC NULLS FIRST, other_chan_sales_price#84 ASC NULLS FIRST, ratio#78 ASC NULLS FIRST], [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84] +(49) CometFilter +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) + +(50) CometProject +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] + +(51) CometExchange +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(52) CometSort +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] + +(53) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter + +(54) CometFilter +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) + +(55) CometProject +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] + +(56) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#50, d_year#51] + +(57) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight + +(58) CometProject +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] + +(59) CometHashAggregate +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] + +(60) CometExchange +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(61) CometHashAggregate +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] + +(62) CometFilter +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) + +(63) CometSort +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] + +(64) CometSortMergeJoin +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner + +(65) CometColumnarToRow [codegen id : 1] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(66) Project [codegen id : 1] +Output [13]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(67) TakeOrderedAndProject +Input [13]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: 100, [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, ratio#60 ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/extended.txt new file mode 100644 index 0000000000..f180fb1878 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/extended.txt @@ -0,0 +1,73 @@ + TakeOrderedAndProject [COMET: ] ++- Project [COMET: Comet does not support Spark's BigDecimal rounding] + +- CometColumnarToRow + +- CometSortMergeJoin + :- CometProject + : +- CometSortMergeJoin + : :- CometSort + : : +- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometFilter + : : : +- CometSortMergeJoin + : : : :- CometSort + : : : : +- CometExchange + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : +- CometSort + : : : +- CometExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : +- CometBroadcastExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometSort + : +- CometFilter + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometFilter + : : +- CometSortMergeJoin + : : :- CometSort + : : : +- CometExchange + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometSort + : : +- CometExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_returns + : +- CometBroadcastExchange + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometSort + +- CometFilter + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometFilter + : +- CometSortMergeJoin + : :- CometSort + : : +- CometExchange + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : +- CometSort + : +- CometExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.catalog_returns + +- CometBroadcastExchange + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 68 out of 70 eligible operators (97%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/simplified.txt index a9b6bf9fd3..5b555777a1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/simplified.txt @@ -1,123 +1,69 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (23) + WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] + CometColumnarToRow InputAdapter - WholeStageCodegen (15) - Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] - InputAdapter - WholeStageCodegen (7) - Sort [ss_sold_year,ss_item_sk,ss_customer_sk] - HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - Filter [sr_ticket_number] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (14) - Sort [ws_sold_year,ws_item_sk,ws_customer_sk] - Filter [ws_qty] - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - WholeStageCodegen (13) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - Filter [wr_order_number] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (9) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #6 - WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (11) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #7 - WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 - InputAdapter - WholeStageCodegen (22) - Sort [cs_sold_year,cs_item_sk,cs_customer_sk] - Filter [cs_qty] - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - WholeStageCodegen (21) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - Filter [cr_order_number] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (17) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #9 - WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (19) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #10 - WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 + CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/explain.txt index 866e41848c..d57cf3c7f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/explain.txt @@ -1,716 +1,646 @@ == Physical Plan == -TakeOrderedAndProject (124) -+- * HashAggregate (123) - +- Exchange (122) - +- * HashAggregate (121) - +- Union (120) - :- * HashAggregate (109) - : +- Exchange (108) - : +- * HashAggregate (107) - : +- Union (106) - : :- * HashAggregate (43) - : : +- Exchange (42) - : : +- * HashAggregate (41) - : : +- * Project (40) - : : +- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Project (33) - : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : :- * Project (26) - : : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : : :- * Project (20) - : : : : : +- * BroadcastHashJoin Inner BuildRight (19) - : : : : : :- * Project (13) - : : : : : : +- * SortMergeJoin LeftOuter (12) - : : : : : : :- * Sort (5) - : : : : : : : +- Exchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- * Sort (11) - : : : : : : +- Exchange (10) - : : : : : : +- * Project (9) - : : : : : : +- * Filter (8) - : : : : : : +- * ColumnarToRow (7) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : : : : +- BroadcastExchange (18) - : : : : : +- * Project (17) - : : : : : +- * Filter (16) - : : : : : +- * ColumnarToRow (15) - : : : : : +- Scan parquet spark_catalog.default.date_dim (14) - : : : : +- BroadcastExchange (24) - : : : : +- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.store (21) - : : : +- BroadcastExchange (31) - : : : +- * Project (30) - : : : +- * Filter (29) - : : : +- * ColumnarToRow (28) - : : : +- Scan parquet spark_catalog.default.item (27) - : : +- BroadcastExchange (38) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.promotion (34) - : :- * HashAggregate (74) - : : +- Exchange (73) - : : +- * HashAggregate (72) - : : +- * Project (71) - : : +- * BroadcastHashJoin Inner BuildRight (70) - : : :- * Project (68) - : : : +- * BroadcastHashJoin Inner BuildRight (67) - : : : :- * Project (65) - : : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : : :- * Project (59) - : : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : : :- * Project (56) - : : : : : : +- * SortMergeJoin LeftOuter (55) - : : : : : : :- * Sort (48) - : : : : : : : +- Exchange (47) - : : : : : : : +- * Filter (46) - : : : : : : : +- * ColumnarToRow (45) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (44) - : : : : : : +- * Sort (54) - : : : : : : +- Exchange (53) - : : : : : : +- * Project (52) - : : : : : : +- * Filter (51) - : : : : : : +- * ColumnarToRow (50) - : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (49) - : : : : : +- ReusedExchange (57) - : : : : +- BroadcastExchange (63) - : : : : +- * Filter (62) - : : : : +- * ColumnarToRow (61) - : : : : +- Scan parquet spark_catalog.default.catalog_page (60) - : : : +- ReusedExchange (66) - : : +- ReusedExchange (69) - : +- * HashAggregate (105) - : +- Exchange (104) - : +- * HashAggregate (103) - : +- * Project (102) - : +- * BroadcastHashJoin Inner BuildRight (101) - : :- * Project (99) - : : +- * BroadcastHashJoin Inner BuildRight (98) - : : :- * Project (96) - : : : +- * BroadcastHashJoin Inner BuildRight (95) - : : : :- * Project (90) - : : : : +- * BroadcastHashJoin Inner BuildRight (89) - : : : : :- * Project (87) - : : : : : +- * SortMergeJoin LeftOuter (86) - : : : : : :- * Sort (79) - : : : : : : +- Exchange (78) - : : : : : : +- * Filter (77) - : : : : : : +- * ColumnarToRow (76) - : : : : : : +- Scan parquet spark_catalog.default.web_sales (75) - : : : : : +- * Sort (85) - : : : : : +- Exchange (84) - : : : : : +- * Project (83) - : : : : : +- * Filter (82) - : : : : : +- * ColumnarToRow (81) - : : : : : +- Scan parquet spark_catalog.default.web_returns (80) - : : : : +- ReusedExchange (88) - : : : +- BroadcastExchange (94) - : : : +- * Filter (93) - : : : +- * ColumnarToRow (92) - : : : +- Scan parquet spark_catalog.default.web_site (91) - : : +- ReusedExchange (97) - : +- ReusedExchange (100) - :- * HashAggregate (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- ReusedExchange (110) - +- * HashAggregate (119) - +- Exchange (118) - +- * HashAggregate (117) - +- * HashAggregate (116) - +- ReusedExchange (115) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (116) ++- CometTakeOrderedAndProject (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometHashAggregate (100) + : +- CometExchange (99) + : +- CometHashAggregate (98) + : +- CometUnion (97) + : :- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (29) + : : : +- CometBroadcastHashJoin (28) + : : : :- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometProject (17) + : : : : : +- CometBroadcastHashJoin (16) + : : : : : :- CometProject (11) + : : : : : : +- CometSortMergeJoin (10) + : : : : : : :- CometSort (4) + : : : : : : : +- CometExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometSort (9) + : : : : : : +- CometExchange (8) + : : : : : : +- CometProject (7) + : : : : : : +- CometFilter (6) + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns (5) + : : : : : +- CometBroadcastExchange (15) + : : : : : +- CometProject (14) + : : : : : +- CometFilter (13) + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim (12) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometNativeScan parquet spark_catalog.default.store (18) + : : : +- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometNativeScan parquet spark_catalog.default.item (24) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometNativeScan parquet spark_catalog.default.promotion (30) + : :- CometHashAggregate (67) + : : +- CometExchange (66) + : : +- CometHashAggregate (65) + : : +- CometProject (64) + : : +- CometBroadcastHashJoin (63) + : : :- CometProject (61) + : : : +- CometBroadcastHashJoin (60) + : : : :- CometProject (58) + : : : : +- CometBroadcastHashJoin (57) + : : : : :- CometProject (52) + : : : : : +- CometBroadcastHashJoin (51) + : : : : : :- CometProject (49) + : : : : : : +- CometSortMergeJoin (48) + : : : : : : :- CometSort (42) + : : : : : : : +- CometExchange (41) + : : : : : : : +- CometFilter (40) + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales (39) + : : : : : : +- CometSort (47) + : : : : : : +- CometExchange (46) + : : : : : : +- CometProject (45) + : : : : : : +- CometFilter (44) + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns (43) + : : : : : +- ReusedExchange (50) + : : : : +- CometBroadcastExchange (56) + : : : : +- CometProject (55) + : : : : +- CometFilter (54) + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page (53) + : : : +- ReusedExchange (59) + : : +- ReusedExchange (62) + : +- CometHashAggregate (96) + : +- CometExchange (95) + : +- CometHashAggregate (94) + : +- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometProject (81) + : : : : +- CometBroadcastHashJoin (80) + : : : : :- CometProject (78) + : : : : : +- CometSortMergeJoin (77) + : : : : : :- CometSort (71) + : : : : : : +- CometExchange (70) + : : : : : : +- CometFilter (69) + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales (68) + : : : : : +- CometSort (76) + : : : : : +- CometExchange (75) + : : : : : +- CometProject (74) + : : : : : +- CometFilter (73) + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns (72) + : : : : +- ReusedExchange (79) + : : : +- CometBroadcastExchange (85) + : : : +- CometProject (84) + : : : +- CometFilter (83) + : : : +- CometNativeScan parquet spark_catalog.default.web_site (82) + : : +- ReusedExchange (88) + : +- ReusedExchange (91) + :- CometHashAggregate (105) + : +- CometExchange (104) + : +- CometHashAggregate (103) + : +- CometHashAggregate (102) + : +- ReusedExchange (101) + +- CometHashAggregate (110) + +- CometExchange (109) + +- CometHashAggregate (108) + +- CometHashAggregate (107) + +- ReusedExchange (106) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometNativeScan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +(7) CometProject Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -(10) Exchange +(8) CometExchange Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] -Right keys [2]: [sr_item_sk#8, sr_ticket_number#9] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter -(13) Project [codegen id : 9] -Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +(11) CometProject Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(14) Scan parquet spark_catalog.default.date_dim +(12) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] - -(16) Filter [codegen id : 5] +(13) CometFilter Input [2]: [d_date_sk#13, d_date#14] Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) -(17) Project [codegen id : 5] -Output [1]: [d_date_sk#13] +(14) CometProject Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(18) BroadcastExchange +(15) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#13] -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight -(20) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +(17) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(21) Scan parquet spark_catalog.default.store +(18) CometNativeScan parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] - -(23) Filter [codegen id : 6] +(19) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(24) BroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] -(26) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] +(22) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(27) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometNativeScan parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] - -(29) Filter [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) -(30) Project [codegen id : 7] -Output [1]: [i_item_sk#17] -Input [2]: [i_item_sk#17, i_current_price#18] +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] -(31) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(33) Project [codegen id : 9] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(34) Scan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] +(30) CometNativeScan parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) -(36) Filter [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] -(37) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] -Input [2]: [p_promo_sk#19, p_channel_tv#20] +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] -(38) BroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(40) Project [codegen id : 9] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] - -(41) HashAggregate [codegen id : 9] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -(42) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(43) HashAggregate [codegen id : 10] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] +(38) CometHashAggregate +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [store channel AS channel#34, concat(store, s_store_id#16) AS id#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#38] -(44) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(39) CometNativeScan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(46) Filter [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) +(40) CometFilter +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Condition : ((isnotnull(cs_catalog_page_sk#27) AND isnotnull(cs_item_sk#28)) AND isnotnull(cs_promo_sk#29)) -(47) Exchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(41) CometExchange +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: hashpartitioning(cs_item_sk#28, cs_order_number#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(48) Sort [codegen id : 12] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST], false, 0 +(42) CometSort +Input [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33], [cs_item_sk#28 ASC NULLS FIRST, cs_order_number#30 ASC NULLS FIRST] -(49) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(43) CometNativeScan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(44) CometFilter +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Condition : (isnotnull(cr_item_sk#34) AND isnotnull(cr_order_number#35)) -(51) Filter [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Condition : (isnotnull(cr_item_sk#46) AND isnotnull(cr_order_number#47)) +(45) CometProject +Input [5]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37, cr_returned_date_sk#38] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] -(52) Project [codegen id : 13] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(46) CometExchange +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: hashpartitioning(cr_item_sk#34, cr_order_number#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(53) Exchange -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: hashpartitioning(cr_item_sk#46, cr_order_number#47, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(47) CometSort +Input [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37], [cr_item_sk#34 ASC NULLS FIRST, cr_order_number#35 ASC NULLS FIRST] -(54) Sort [codegen id : 14] -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST], false, 0 +(48) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [4]: [cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_item_sk#28, cs_order_number#30], [cr_item_sk#34, cr_order_number#35], LeftOuter -(55) SortMergeJoin [codegen id : 19] -Left keys [2]: [cs_item_sk#40, cs_order_number#42] -Right keys [2]: [cr_item_sk#46, cr_order_number#47] -Join type: LeftOuter -Join condition: None +(49) CometProject +Input [11]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_order_number#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_item_sk#34, cr_order_number#35, cr_return_amount#36, cr_net_loss#37] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] -(56) Project [codegen id : 19] -Output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] +(50) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#39] -(57) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#51] +(51) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37] +Right output [1]: [d_date_sk#39] +Arguments: [cs_sold_date_sk#33], [d_date_sk#39], Inner, BuildRight -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None +(52) CometProject +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, cr_return_amount#36, cr_net_loss#37, d_date_sk#39] +Arguments: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37], [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] -(59) Project [codegen id : 19] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#51] - -(60) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] +(53) CometNativeScan parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(62) Filter [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Condition : isnotnull(cp_catalog_page_sk#52) - -(63) BroadcastExchange -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(64) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#52] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 19] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(66) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#54] - -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#54] -Join type: Inner -Join condition: None - -(68) Project [codegen id : 19] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, i_item_sk#54] - -(69) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#55] - -(70) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(71) Project [codegen id : 19] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, p_promo_sk#55] - -(72) HashAggregate [codegen id : 19] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Results [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] - -(73) Exchange -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) HashAggregate [codegen id : 20] -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#66, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68] -Results [5]: [catalog channel AS channel#69, concat(catalog_page, cp_catalog_page_id#53) AS id#70, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#66,17,2) AS sales#71, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67 AS returns#72, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68 AS profit#73] - -(75) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] +(54) CometFilter +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Condition : isnotnull(cp_catalog_page_sk#40) + +(55) CometProject +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#41] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42], [cp_catalog_page_sk#40, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#41, 16, true, false, true) AS cp_catalog_page_id#42] + +(56) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cp_catalog_page_sk#40, cp_catalog_page_id#42] + +(57) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37] +Right output [2]: [cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_catalog_page_sk#27], [cp_catalog_page_sk#40], Inner, BuildRight + +(58) CometProject +Input [9]: [cs_catalog_page_sk#27, cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_sk#40, cp_catalog_page_id#42] +Arguments: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(59) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#43] + +(60) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [i_item_sk#43] +Arguments: [cs_item_sk#28], [i_item_sk#43], Inner, BuildRight + +(61) CometProject +Input [8]: [cs_item_sk#28, cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, i_item_sk#43] +Arguments: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(62) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#44] + +(63) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Right output [1]: [p_promo_sk#44] +Arguments: [cs_promo_sk#29], [p_promo_sk#44], Inner, BuildRight + +(64) CometProject +Input [7]: [cs_promo_sk#29, cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42, p_promo_sk#44] +Arguments: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42], [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] + +(65) CometHashAggregate +Input [5]: [cs_ext_sales_price#31, cs_net_profit#32, cr_return_amount#36, cr_net_loss#37, cp_catalog_page_id#42] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(66) CometExchange +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Arguments: hashpartitioning(cp_catalog_page_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(67) CometHashAggregate +Input [6]: [cp_catalog_page_id#42, sum#45, sum#46, isEmpty#47, sum#48, isEmpty#49] +Keys [1]: [cp_catalog_page_id#42] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum((cs_net_profit#32 - coalesce(cast(cr_net_loss#37 as decimal(12,2)), 0.00)))] + +(68) CometNativeScan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#80)] +Location: InMemoryFileIndex(0 paths)[] +PartitionFilters: [isnotnull(ws_sold_date_sk#56)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] +(69) CometFilter +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Condition : ((isnotnull(ws_web_site_sk#51) AND isnotnull(ws_item_sk#50)) AND isnotnull(ws_promo_sk#52)) -(77) Filter [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Condition : ((isnotnull(ws_web_site_sk#75) AND isnotnull(ws_item_sk#74)) AND isnotnull(ws_promo_sk#76)) +(70) CometExchange +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: hashpartitioning(ws_item_sk#50, ws_order_number#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(78) Exchange -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: hashpartitioning(ws_item_sk#74, ws_order_number#77, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(71) CometSort +Input [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56], [ws_item_sk#50 ASC NULLS FIRST, ws_order_number#53 ASC NULLS FIRST] -(79) Sort [codegen id : 22] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: [ws_item_sk#74 ASC NULLS FIRST, ws_order_number#77 ASC NULLS FIRST], false, 0 - -(80) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(72) CometNativeScan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(81) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] - -(82) Filter [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Condition : (isnotnull(wr_item_sk#81) AND isnotnull(wr_order_number#82)) +(73) CometFilter +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Condition : (isnotnull(wr_item_sk#57) AND isnotnull(wr_order_number#58)) -(83) Project [codegen id : 23] -Output [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(74) CometProject +Input [5]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60, wr_returned_date_sk#61] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] -(84) Exchange -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: hashpartitioning(wr_item_sk#81, wr_order_number#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(75) CometExchange +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: hashpartitioning(wr_item_sk#57, wr_order_number#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(85) Sort [codegen id : 24] -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: [wr_item_sk#81 ASC NULLS FIRST, wr_order_number#82 ASC NULLS FIRST], false, 0 +(76) CometSort +Input [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60], [wr_item_sk#57 ASC NULLS FIRST, wr_order_number#58 ASC NULLS FIRST] -(86) SortMergeJoin [codegen id : 29] -Left keys [2]: [ws_item_sk#74, ws_order_number#77] -Right keys [2]: [wr_item_sk#81, wr_order_number#82] -Join type: LeftOuter -Join condition: None +(77) CometSortMergeJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56] +Right output [4]: [wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_order_number#53], [wr_item_sk#57, wr_order_number#58], LeftOuter -(87) Project [codegen id : 29] -Output [8]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84] -Input [11]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] +(78) CometProject +Input [11]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_order_number#53, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_item_sk#57, wr_order_number#58, wr_return_amt#59, wr_net_loss#60] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] -(88) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#86] +(79) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#62] -(89) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_sold_date_sk#80] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None +(80) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60] +Right output [1]: [d_date_sk#62] +Arguments: [ws_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight -(90) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84, d_date_sk#86] +(81) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, ws_sold_date_sk#56, wr_return_amt#59, wr_net_loss#60, d_date_sk#62] +Arguments: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60], [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] -(91) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#87, web_site_id#88] +(82) CometNativeScan parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#63, web_site_id#64] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(92) ColumnarToRow [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] - -(93) Filter [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] -Condition : isnotnull(web_site_sk#87) - -(94) BroadcastExchange -Input [2]: [web_site_sk#87, web_site_id#88] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_web_site_sk#75] -Right keys [1]: [web_site_sk#87] -Join type: Inner -Join condition: None - -(96) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_sk#87, web_site_id#88] - -(97) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#89] - -(98) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_item_sk#74] -Right keys [1]: [i_item_sk#89] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 29] -Output [6]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [8]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, i_item_sk#89] - -(100) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#90] - -(101) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_promo_sk#76] -Right keys [1]: [p_promo_sk#90] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 29] -Output [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [7]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, p_promo_sk#90] - -(103) HashAggregate [codegen id : 29] -Input [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Keys [1]: [web_site_id#88] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#78)), partial_sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Results [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] - -(104) Exchange -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(web_site_id#88, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(105) HashAggregate [codegen id : 30] -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Keys [1]: [web_site_id#88] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#78)), sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#78))#101, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103] -Results [5]: [web channel AS channel#104, concat(web_site, web_site_id#88) AS id#105, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#78))#101,17,2) AS sales#106, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102 AS returns#107, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103 AS profit#108] - -(106) Union - -(107) HashAggregate [codegen id : 31] -Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] -Keys [2]: [channel#34, id#35] -Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] -Aggregate Attributes [6]: [sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Results [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(108) Exchange -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(109) HashAggregate [codegen id : 32] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [5]: [channel#34, id#35, cast(sum(sales#36)#121 as decimal(37,2)) AS sales#124, cast(sum(returns#37)#122 as decimal(38,2)) AS returns#125, cast(sum(profit#38)#123 as decimal(38,2)) AS profit#126] - -(110) ReusedExchange [Reuses operator id: 108] -Output [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(111) HashAggregate [codegen id : 64] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [4]: [channel#34, sum(sales#36)#121 AS sales#127, sum(returns#37)#122 AS returns#128, sum(profit#38)#123 AS profit#129] - -(112) HashAggregate [codegen id : 64] -Input [4]: [channel#34, sales#127, returns#128, profit#129] -Keys [1]: [channel#34] -Functions [3]: [partial_sum(sales#127), partial_sum(returns#128), partial_sum(profit#129)] -Aggregate Attributes [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Results [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] - -(113) Exchange -Input [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(114) HashAggregate [codegen id : 65] -Input [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Keys [1]: [channel#34] -Functions [3]: [sum(sales#127), sum(returns#128), sum(profit#129)] -Aggregate Attributes [3]: [sum(sales#127)#142, sum(returns#128)#143, sum(profit#129)#144] -Results [5]: [channel#34, null AS id#145, sum(sales#127)#142 AS sales#146, sum(returns#128)#143 AS returns#147, sum(profit#129)#144 AS profit#148] - -(115) ReusedExchange [Reuses operator id: 108] -Output [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(116) HashAggregate [codegen id : 97] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [3]: [sum(sales#36)#121 AS sales#127, sum(returns#37)#122 AS returns#128, sum(profit#38)#123 AS profit#129] - -(117) HashAggregate [codegen id : 97] -Input [3]: [sales#127, returns#128, profit#129] +(83) CometFilter +Input [2]: [web_site_sk#63, web_site_id#64] +Condition : isnotnull(web_site_sk#63) + +(84) CometProject +Input [2]: [web_site_sk#63, web_site_id#64] +Arguments: [web_site_sk#63, web_site_id#65], [web_site_sk#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#64, 16, true, false, true) AS web_site_id#65] + +(85) CometBroadcastExchange +Input [2]: [web_site_sk#63, web_site_id#65] +Arguments: [web_site_sk#63, web_site_id#65] + +(86) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60] +Right output [2]: [web_site_sk#63, web_site_id#65] +Arguments: [ws_web_site_sk#51], [web_site_sk#63], Inner, BuildRight + +(87) CometProject +Input [9]: [ws_item_sk#50, ws_web_site_sk#51, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_sk#63, web_site_id#65] +Arguments: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(88) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#66] + +(89) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [i_item_sk#66] +Arguments: [ws_item_sk#50], [i_item_sk#66], Inner, BuildRight + +(90) CometProject +Input [8]: [ws_item_sk#50, ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, i_item_sk#66] +Arguments: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(91) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#67] + +(92) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Right output [1]: [p_promo_sk#67] +Arguments: [ws_promo_sk#52], [p_promo_sk#67], Inner, BuildRight + +(93) CometProject +Input [7]: [ws_promo_sk#52, ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65, p_promo_sk#67] +Arguments: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65], [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] + +(94) CometHashAggregate +Input [5]: [ws_ext_sales_price#54, ws_net_profit#55, wr_return_amt#59, wr_net_loss#60, web_site_id#65] +Keys [1]: [web_site_id#65] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#54)), partial_sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(95) CometExchange +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Arguments: hashpartitioning(web_site_id#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(96) CometHashAggregate +Input [6]: [web_site_id#65, sum#68, sum#69, isEmpty#70, sum#71, isEmpty#72] +Keys [1]: [web_site_id#65] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#54)), sum(coalesce(cast(wr_return_amt#59 as decimal(12,2)), 0.00)), sum((ws_net_profit#55 - coalesce(cast(wr_net_loss#60 as decimal(12,2)), 0.00)))] + +(97) CometUnion +Child 0 Input [5]: [channel#73, id#74, sales#75, returns#76, profit#77] +Child 1 Input [5]: [channel#78, id#79, sales#80, returns#81, profit#82] +Child 2 Input [5]: [channel#83, id#84, sales#85, returns#86, profit#87] + +(98) CometHashAggregate +Input [5]: [channel#73, id#74, sales#75, returns#76, profit#77] +Keys [2]: [channel#73, id#74] +Functions [3]: [partial_sum(sales#75), partial_sum(returns#76), partial_sum(profit#77)] + +(99) CometExchange +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Arguments: hashpartitioning(channel#73, id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(100) CometHashAggregate +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#73, id#74] +Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] + +(101) ReusedExchange [Reuses operator id: 99] +Output [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(102) CometHashAggregate +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#73, id#74] +Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] + +(103) CometHashAggregate +Input [4]: [channel#73, sales#94, returns#95, profit#96] +Keys [1]: [channel#73] +Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] + +(104) CometExchange +Input [7]: [channel#73, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Arguments: hashpartitioning(channel#73, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(105) CometHashAggregate +Input [7]: [channel#73, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Keys [1]: [channel#73] +Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] + +(106) ReusedExchange [Reuses operator id: 99] +Output [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(107) CometHashAggregate +Input [8]: [channel#73, id#74, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#73, id#74] +Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] + +(108) CometHashAggregate +Input [3]: [sales#94, returns#95, profit#96] Keys: [] -Functions [3]: [partial_sum(sales#127), partial_sum(returns#128), partial_sum(profit#129)] -Aggregate Attributes [6]: [sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] -Results [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] +Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] -(118) Exchange -Input [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] +(109) CometExchange +Input [6]: [sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(119) HashAggregate [codegen id : 98] -Input [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] +(110) CometHashAggregate +Input [6]: [sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] Keys: [] -Functions [3]: [sum(sales#127), sum(returns#128), sum(profit#129)] -Aggregate Attributes [3]: [sum(sales#127)#161, sum(returns#128)#162, sum(profit#129)#163] -Results [5]: [null AS channel#164, null AS id#165, sum(sales#127)#161 AS sales#166, sum(returns#128)#162 AS returns#167, sum(profit#129)#163 AS profit#168] +Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] -(120) Union +(111) CometUnion +Child 0 Input [5]: [channel#73, id#74, sales#109, returns#110, profit#111] +Child 1 Input [5]: [channel#73, id#112, sales#113, returns#114, profit#115] +Child 2 Input [5]: [channel#116, id#117, sales#118, returns#119, profit#120] -(121) HashAggregate [codegen id : 99] -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Keys [5]: [channel#34, id#35, sales#124, returns#125, profit#126] +(112) CometHashAggregate +Input [5]: [channel#73, id#74, sales#109, returns#110, profit#111] +Keys [5]: [channel#73, id#74, sales#109, returns#110, profit#111] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -(122) Exchange -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Arguments: hashpartitioning(channel#34, id#35, sales#124, returns#125, profit#126, 5), ENSURE_REQUIREMENTS, [plan_id=19] +(113) CometExchange +Input [5]: [channel#73, id#74, sales#109, returns#110, profit#111] +Arguments: hashpartitioning(channel#73, id#74, sales#109, returns#110, profit#111, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(123) HashAggregate [codegen id : 100] -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Keys [5]: [channel#34, id#35, sales#124, returns#125, profit#126] +(114) CometHashAggregate +Input [5]: [channel#73, id#74, sales#109, returns#110, profit#111] +Keys [5]: [channel#73, id#74, sales#109, returns#110, profit#111] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -(124) TakeOrderedAndProject -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#124, returns#125, profit#126] +(115) CometTakeOrderedAndProject +Input [5]: [channel#73, id#74, sales#109, returns#110, profit#111] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#73 ASC NULLS FIRST,id#74 ASC NULLS FIRST], output=[channel#73,id#74,sales#109,returns#110,profit#111]), [channel#73, id#74, sales#109, returns#110, profit#111], 100, 0, [channel#73 ASC NULLS FIRST, id#74 ASC NULLS FIRST], [channel#73, id#74, sales#109, returns#110, profit#111] + +(116) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#73, id#74, sales#109, returns#110, profit#111] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/extended.txt new file mode 100644 index 0000000000..9f0aaa8522 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/extended.txt @@ -0,0 +1,368 @@ +CometColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_site + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometUnion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.store + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : :- CometHashAggregate + : : +- CometExchange + : : +- CometHashAggregate + : : +- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometBroadcastHashJoin + : : : : : :- CometProject + : : : : : : +- CometSortMergeJoin + : : : : : : :- CometSort + : : : : : : : +- CometExchange + : : : : : : : +- CometFilter + : : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : : +- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometProject + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : : +- CometBroadcastExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.item + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.promotion + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_site + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.store_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.store_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.store + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometBroadcastHashJoin + : : : : :- CometProject + : : : : : +- CometSortMergeJoin + : : : : : :- CometSort + : : : : : : +- CometExchange + : : : : : : +- CometFilter + : : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_sales + : : : : : +- CometSort + : : : : : +- CometExchange + : : : : : +- CometProject + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.catalog_returns + : : : : +- CometBroadcastExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.catalog_page + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.item + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.promotion + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometProject + : : : +- CometBroadcastHashJoin + : : : :- CometProject + : : : : +- CometSortMergeJoin + : : : : :- CometSort + : : : : : +- CometExchange + : : : : : +- CometFilter + : : : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : : : +- CometSort + : : : : +- CometExchange + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.web_returns + : : : +- CometBroadcastExchange + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_site + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.promotion + +Comet accelerated 365 out of 365 eligible operators (100%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/simplified.txt index 011fa1eb28..4bdc3ca774 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/simplified.txt @@ -1,203 +1,118 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (100) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (99) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (32) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (31) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (10) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #3 - WholeStageCodegen (9) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #4 - WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #5 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (20) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #10 - WholeStageCodegen (19) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (12) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #11 - WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (14) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (30) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #14 - WholeStageCodegen (29) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - WholeStageCodegen (22) - Sort [ws_item_sk,ws_order_number] - InputAdapter - Exchange [ws_item_sk,ws_order_number] #15 - WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (24) - Sort [wr_item_sk,wr_order_number] - InputAdapter - Exchange [wr_item_sk,wr_order_number] #16 - WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (65) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #18 - WholeStageCodegen (64) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (98) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #19 - WholeStageCodegen (97) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometExchange [channel,id,sales,returns,profit] #1 + CometHashAggregate [channel,id,sales,returns,profit] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel,id] #2 + CometHashAggregate [sales,returns,profit] [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [channel,id,sales,returns,profit] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,s_store_id,sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00)))] + CometExchange [s_store_id] #3 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometNativeScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometNativeScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,cp_catalog_page_id,sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00)))] + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometNativeScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + CometHashAggregate [sum,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,web_site_id,sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00)))] + CometExchange [web_site_id] #14 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometNativeScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometNativeScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange [channel] #18 + CometHashAggregate [sales,returns,profit] [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [channel,id,sales,returns,profit,sum(sales),sum(returns),sum(profit)] + CometExchange #19 + CometHashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometHashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sales,returns,profit,sum(sales),sum(returns),sum(profit)] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/explain.txt index 348af132d6..5d6632e6c6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/explain.txt @@ -1,230 +1,208 @@ == Physical Plan == -TakeOrderedAndProject (38) -+- * Project (37) - +- Window (36) - +- * Sort (35) - +- Exchange (34) - +- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- Union (30) - :- * HashAggregate (19) - : +- Exchange (18) - : +- * HashAggregate (17) - : +- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - :- * HashAggregate (24) - : +- Exchange (23) - : +- * HashAggregate (22) - : +- * HashAggregate (21) - : +- ReusedExchange (20) - +- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * HashAggregate (26) - +- ReusedExchange (25) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (37) ++- * Project (36) + +- Window (35) + +- * CometColumnarToRow (34) + +- CometSort (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometUnion (28) + :- CometHashAggregate (17) + : +- CometExchange (16) + : +- CometHashAggregate (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometNativeScan parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometNativeScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometNativeScan parquet spark_catalog.default.item (9) + :- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometHashAggregate (19) + : +- ReusedExchange (18) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometHashAggregate (24) + +- ReusedExchange (23) + + +(1) CometNativeScan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometNativeScan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ws_item_sk#1, ws_net_paid#2] +(8) CometProject Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] +Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(11) Scan parquet spark_catalog.default.item +(9) CometNativeScan parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] -(16) Project [codegen id : 3] -Output [3]: [ws_net_paid#2, i_class#7, i_category#8] -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] +(13) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#1, ws_net_paid#2] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(17) HashAggregate [codegen id : 3] -Input [3]: [ws_net_paid#2, i_class#7, i_category#8] -Keys [2]: [i_category#8, i_class#7] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] + +(15) CometHashAggregate +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [i_category#8, i_class#7, sum#10] -(18) Exchange -Input [3]: [i_category#8, i_class#7, sum#10] -Arguments: hashpartitioning(i_category#8, i_class#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [3]: [i_category#8, i_class#7, sum#10] -Keys [2]: [i_category#8, i_class#7] +(17) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) as decimal(27,2)) AS total_sum#12, i_category#8, i_class#7, 0 AS g_category#13, 0 AS g_class#14, 0 AS lochierarchy#15] -(20) ReusedExchange [Reuses operator id: 18] -Output [3]: [i_category#8, i_class#7, sum#16] +(18) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#12] -(21) HashAggregate [codegen id : 8] -Input [3]: [i_category#8, i_class#7, sum#16] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) AS total_sum#17, i_category#8] - -(22) HashAggregate [codegen id : 8] -Input [2]: [total_sum#17, i_category#8] -Keys [1]: [i_category#8] -Functions [1]: [partial_sum(total_sum#17)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [3]: [i_category#8, sum#20, isEmpty#21] - -(23) Exchange -Input [3]: [i_category#8, sum#20, isEmpty#21] -Arguments: hashpartitioning(i_category#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) HashAggregate [codegen id : 9] -Input [3]: [i_category#8, sum#20, isEmpty#21] -Keys [1]: [i_category#8] -Functions [1]: [sum(total_sum#17)] -Aggregate Attributes [1]: [sum(total_sum#17)#22] -Results [6]: [sum(total_sum#17)#22 AS total_sum#23, i_category#8, null AS i_class#24, 0 AS g_category#25, 1 AS g_class#26, 1 AS lochierarchy#27] - -(25) ReusedExchange [Reuses operator id: 18] -Output [3]: [i_category#8, i_class#7, sum#28] - -(26) HashAggregate [codegen id : 13] -Input [3]: [i_category#8, i_class#7, sum#28] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) AS total_sum#17] +(19) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#12] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#13))] + +(20) CometHashAggregate +Input [2]: [total_sum#14, i_category#10] +Keys [1]: [i_category#10] +Functions [1]: [partial_sum(total_sum#14)] + +(21) CometExchange +Input [3]: [i_category#10, sum#15, isEmpty#16] +Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(22) CometHashAggregate +Input [3]: [i_category#10, sum#15, isEmpty#16] +Keys [1]: [i_category#10] +Functions [1]: [sum(total_sum#14)] -(27) HashAggregate [codegen id : 13] -Input [1]: [total_sum#17] +(23) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#17] + +(24) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#17] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#18))] + +(25) CometHashAggregate +Input [1]: [total_sum#14] Keys: [] -Functions [1]: [partial_sum(total_sum#17)] -Aggregate Attributes [2]: [sum#29, isEmpty#30] -Results [2]: [sum#31, isEmpty#32] +Functions [1]: [partial_sum(total_sum#14)] -(28) Exchange -Input [2]: [sum#31, isEmpty#32] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [2]: [sum#19, isEmpty#20] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(29) HashAggregate [codegen id : 14] -Input [2]: [sum#31, isEmpty#32] +(27) CometHashAggregate +Input [2]: [sum#19, isEmpty#20] Keys: [] -Functions [1]: [sum(total_sum#17)] -Aggregate Attributes [1]: [sum(total_sum#17)#33] -Results [6]: [sum(total_sum#17)#33 AS total_sum#34, null AS i_category#35, null AS i_class#36, 1 AS g_category#37, 1 AS g_class#38, 2 AS lochierarchy#39] +Functions [1]: [sum(total_sum#14)] -(30) Union +(28) CometUnion +Child 0 Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Child 1 Input [6]: [total_sum#27, i_category#10, i_class#28, g_category#29, g_class#30, lochierarchy#31] +Child 2 Input [6]: [total_sum#32, i_category#33, i_class#34, g_category#35, g_class#36, lochierarchy#37] -(31) HashAggregate [codegen id : 15] -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Keys [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] +(29) CometHashAggregate +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Keys [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] Functions: [] -Aggregate Attributes: [] -Results [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -(32) Exchange -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Arguments: hashpartitioning(total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(30) CometExchange +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Arguments: hashpartitioning(total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(33) HashAggregate [codegen id : 16] -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Keys [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] +(31) CometHashAggregate +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Keys [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, CASE WHEN (g_class#14 = 0) THEN i_category#8 END AS _w0#40] -(34) Exchange -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: hashpartitioning(lochierarchy#15, _w0#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(32) CometExchange +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: hashpartitioning(lochierarchy#26, _w0#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(33) CometSort +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38], [lochierarchy#26 ASC NULLS FIRST, _w0#38 ASC NULLS FIRST, total_sum#21 DESC NULLS LAST] -(35) Sort [codegen id : 17] -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: [lochierarchy#15 ASC NULLS FIRST, _w0#40 ASC NULLS FIRST, total_sum#12 DESC NULLS LAST], false, 0 +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] -(36) Window -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: [rank(total_sum#12) windowspecdefinition(lochierarchy#15, _w0#40, total_sum#12 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#41], [lochierarchy#15, _w0#40], [total_sum#12 DESC NULLS LAST] +(35) Window +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: [rank(total_sum#21) windowspecdefinition(lochierarchy#26, _w0#38, total_sum#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#39], [lochierarchy#26, _w0#38], [total_sum#21 DESC NULLS LAST] -(37) Project [codegen id : 18] -Output [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] -Input [6]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40, rank_within_parent#41] +(36) Project [codegen id : 2] +Output [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] +Input [6]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38, rank_within_parent#39] -(38) TakeOrderedAndProject -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] -Arguments: 100, [lochierarchy#15 DESC NULLS LAST, CASE WHEN (lochierarchy#15 = 0) THEN i_category#8 END ASC NULLS FIRST, rank_within_parent#41 ASC NULLS FIRST], [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] +(37) TakeOrderedAndProject +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] +Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN i_category#22 END ASC NULLS FIRST, rank_within_parent#39 ASC NULLS FIRST], [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/extended.txt new file mode 100644 index 0000000000..fa549246d9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/extended.txt @@ -0,0 +1,69 @@ + TakeOrderedAndProject [COMET: ] ++- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometUnion + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + :- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometHashAggregate + : +- CometExchange + : +- CometHashAggregate + : +- CometProject + : +- CometBroadcastHashJoin + : :- CometProject + : : +- CometBroadcastHashJoin + : : :- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.web_sales + : : +- CometBroadcastExchange + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.date_dim + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.web_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.date_dim + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.item + +Comet accelerated 63 out of 66 eligible operators (95%). Final plan contains 1 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/simplified.txt index db47334fd5..d86038290c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/simplified.txt @@ -1,64 +1,41 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (18) + WholeStageCodegen (2) Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (17) - Sort [lochierarchy,_w0,total_sum] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (16) - HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 - WholeStageCodegen (15) - HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] - InputAdapter - Union - WholeStageCodegen (4) - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,g_category,g_class,lochierarchy,sum] - InputAdapter - Exchange [i_category,i_class] #3 - WholeStageCodegen (3) - HashAggregate [i_category,i_class,ws_net_paid] [sum,sum] - Project [ws_net_paid,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_net_paid] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - WholeStageCodegen (9) - HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - InputAdapter - Exchange [i_category] #6 - WholeStageCodegen (8) - HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 - WholeStageCodegen (14) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - InputAdapter - Exchange #7 - WholeStageCodegen (13) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 + CometSort [total_sum,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_category] [total_sum,i_category,i_class,lochierarchy,_w0,g_class] + CometExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 + CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + CometUnion [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + CometHashAggregate [sum] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(UnscaledValue(ws_net_paid)),i_category,i_class] + CometExchange [i_category,i_class] #3 + CometHashAggregate [ws_net_paid] [i_category,i_class,sum] + CometProject [ws_net_paid,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometNativeScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] + CometExchange [i_category] #6 + CometHashAggregate [total_sum] [i_category,sum,isEmpty] + CometHashAggregate [i_class,sum] [total_sum,i_category,sum(UnscaledValue(ws_net_paid))] + ReusedExchange [i_category,i_class,sum] #3 + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] + CometExchange #7 + CometHashAggregate [total_sum] [sum,isEmpty] + CometHashAggregate [i_category,i_class,sum] [total_sum,sum(UnscaledValue(ws_net_paid))] + ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/explain.txt index d50ed8e269..1cfd35fde7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/explain.txt @@ -1,145 +1,140 @@ == Physical Plan == -* Sort (25) -+- Exchange (24) - +- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (25) ++- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometNativeScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometNativeScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometNativeScan parquet spark_catalog.default.date_dim (9) + + +(1) CometNativeScan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true -Location: InMemoryFileIndex [] +Location: InMemoryFileIndex(0 paths)[] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometNativeScan parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometNativeScan parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16] -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] + +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] +(23) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(24) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(25) Sort [codegen id : 7] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], true, 0 +(25) CometColumnarToRow [codegen id : 3] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/extended.txt new file mode 100644 index 0000000000..5bbef326d3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/extended.txt @@ -0,0 +1,27 @@ +CometColumnarToRow ++- CometSort + +- CometColumnarExchange + +- Project + +- Window + +- CometColumnarToRow + +- CometSort + +- CometExchange + +- CometHashAggregate + +- CometExchange + +- CometHashAggregate + +- CometProject + +- CometBroadcastHashJoin + :- CometProject + : +- CometBroadcastHashJoin + : :- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store_sales + : +- CometBroadcastExchange + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.item + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.date_dim + +Comet accelerated 21 out of 23 eligible operators (91%). Final plan contains 2 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/simplified.txt index 729dc08617..7acc6e37c6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/simplified.txt @@ -1,41 +1,31 @@ -WholeStageCodegen (7) - Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio] +WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (6) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] + CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 + WholeStageCodegen (2) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometExchange [i_class] #2 + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometNativeScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometNativeScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometNativeScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 67d1c8240b..c5cab02f29 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -63,7 +63,10 @@ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plu */ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBase { protected val scanImpls: Seq[String] = - Seq(CometConf.SCAN_AUTO, CometConf.SCAN_NATIVE_ICEBERG_COMPAT) + Seq( + CometConf.SCAN_AUTO, + CometConf.SCAN_NATIVE_ICEBERG_COMPAT, + CometConf.SCAN_NATIVE_DATAFUSION) protected val baseResourcePath: File = { getWorkspaceFilePath("spark", "src", "test", "resources", "tpcds-plan-stability").toFile @@ -215,12 +218,17 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa s"$tpcdsGroup/$query.sql", classLoader = Thread.currentThread().getContextClassLoader) + // native_datafusion does not support DPP yet + val dppEnabled = + CometConf.COMET_NATIVE_SCAN_IMPL.get(conf) != CometConf.SCAN_NATIVE_DATAFUSION + withSQLConf( CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true", CometConf.COMET_EXPLAIN_VERBOSE_ENABLED.key -> "true", CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", + SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> dppEnabled.toString, CometConf.COMET_DPP_FALLBACK_ENABLED.key -> "false", CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key -> "true",